{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"CH05c_Multi-class_Classification.ipynb","provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyO2hBd/tE18yqAA78qNBNVu"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"GPU","widgets":{"application/vnd.jupyter.widget-state+json":{"21fbbe9bd6d94f308668ac1e4c2a51d0":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_f391632fced34067a6bc7f76034d8d38","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_d9bad87a40624bbe9a7b8e5021b6b878","IPY_MODEL_6f869fe141f847a881532c74c5d8b269"]}},"f391632fced34067a6bc7f76034d8d38":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"d9bad87a40624bbe9a7b8e5021b6b878":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_35a5b5e6ff374b6a80e8677d695a5409","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":262620,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":262620,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_bed0bf19829f4957985091c7f308190c"}},"6f869fe141f847a881532c74c5d8b269":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_43a1a07853414f1fbd41c1d7a7e2a7fd","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 263k/263k [00:01&lt;00:00, 133kB/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_66f31b78d9974a51bd28e01d073b72a7"}},"35a5b5e6ff374b6a80e8677d695a5409":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"bed0bf19829f4957985091c7f308190c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"43a1a07853414f1fbd41c1d7a7e2a7fd":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"66f31b78d9974a51bd28e01d073b72a7":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"8ed902a5a04e4bdaac8f961afe52b622":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_95a51d9d15734391a3895ac76b7a3676","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_395f10a793684087b6e3747154e17f8d","IPY_MODEL_c905e0961cf84af2bec6a29ddc0d4237"]}},"95a51d9d15734391a3895ac76b7a3676":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"395f10a793684087b6e3747154e17f8d":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_b3652deb88d94a3e91bc0fa58b9dce54","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":59,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":59,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_94930151f87e495b974a7f40de2b1db4"}},"c905e0961cf84af2bec6a29ddc0d4237":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_4e66a8a8057e4232a3881aef3ac3afa9","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 59.0/59.0 [00:00&lt;00:00, 730B/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_a098193923044bfc89afdfa4faf8cc25"}},"b3652deb88d94a3e91bc0fa58b9dce54":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"94930151f87e495b974a7f40de2b1db4":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"4e66a8a8057e4232a3881aef3ac3afa9":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"a098193923044bfc89afdfa4faf8cc25":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"eda88132d04243bcb499af9e82a789ba":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_b9f606782e924fdd9a22ee6039ef76f2","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_ce256db990954d9f9782eae9a248b40b","IPY_MODEL_4557e0d687a44a8ab109109094f3ebd6"]}},"b9f606782e924fdd9a22ee6039ef76f2":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"ce256db990954d9f9782eae9a248b40b":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_5db915425bac4d9caba753797e9faf29","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":385,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":385,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_6613484a4d0c477fb4ec5a3d4da81a73"}},"4557e0d687a44a8ab109109094f3ebd6":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_d391715c96a84d078976a10fab20265a","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 385/385 [00:34&lt;00:00, 11.2B/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_0479af3dea114a4b86c98cd104325344"}},"5db915425bac4d9caba753797e9faf29":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"6613484a4d0c477fb4ec5a3d4da81a73":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"d391715c96a84d078976a10fab20265a":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"0479af3dea114a4b86c98cd104325344":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"f8532b3abd7e4243a8fb0ade8e0540e7":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_1886bd7f86534fcea494bef48439e7db","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_85c4358f5a6e411fbff53e3dfe3eb88d","IPY_MODEL_cfe8a0c3575a49b6b298262ed7acf76d"]}},"1886bd7f86534fcea494bef48439e7db":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"85c4358f5a6e411fbff53e3dfe3eb88d":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_4622af58d24f4eea8f67f49364118ccd","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":445018749,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":445018749,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_b42f5e933f494ea69f3dbaac391b5e23"}},"cfe8a0c3575a49b6b298262ed7acf76d":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_f92aca32755f45c3ba2b8cab79d44eed","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 445M/445M [00:09&lt;00:00, 49.2MB/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_dd7628d863794892b9b1fdd7c5e5df42"}},"4622af58d24f4eea8f67f49364118ccd":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"b42f5e933f494ea69f3dbaac391b5e23":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"f92aca32755f45c3ba2b8cab79d44eed":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"dd7628d863794892b9b1fdd7c5e5df42":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}}}}},"cells":[{"cell_type":"markdown","metadata":{"id":"K8rPS5-aTUys"},"source":["# FineTuning BERT for Multi-Class Classification with custom datasets"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"bixw-FrJ-6La","executionInfo":{"status":"ok","timestamp":1625398158764,"user_tz":-180,"elapsed":25729,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"850fe9c3-dc21-4036-bcde-b1b0b5c43034"},"source":["import torch, os\n","from google.colab import drive\n","drive.mount('/content/drive')\n","if os.getcwd() != \"/content/drive/My Drive/akademi/Packt NLP with Transformers/CH05\":\n","  os.chdir(\"drive/MyDrive/akademi/Packt NLP with Transformers/CH05\")"],"execution_count":1,"outputs":[{"output_type":"stream","text":["Mounted at /content/drive\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"6AJ4_0bE678m","executionInfo":{"status":"ok","timestamp":1625398170253,"user_tz":-180,"elapsed":7445,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"a5939dd9-d7bf-4dd0-913c-324e94db7643"},"source":["!pip install transformers datasets"],"execution_count":2,"outputs":[{"output_type":"stream","text":["Collecting transformers\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/fd/1a/41c644c963249fd7f3836d926afa1e3f1cc234a1c40d80c5f03ad8f6f1b2/transformers-4.8.2-py3-none-any.whl (2.5MB)\n","\u001b[K     |████████████████████████████████| 2.5MB 14.6MB/s \n","\u001b[?25hCollecting datasets\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/08/a2/d4e1024c891506e1cee8f9d719d20831bac31cb5b7416983c4d2f65a6287/datasets-1.8.0-py3-none-any.whl (237kB)\n","\u001b[K     |████████████████████████████████| 245kB 56.1MB/s \n","\u001b[?25hCollecting huggingface-hub==0.0.12\n","  Downloading https://files.pythonhosted.org/packages/2f/ee/97e253668fda9b17e968b3f97b2f8e53aa0127e8807d24a547687423fe0b/huggingface_hub-0.0.12-py3-none-any.whl\n","Collecting tokenizers<0.11,>=0.10.1\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/e2/df3543e8ffdab68f5acc73f613de9c2b155ac47f162e725dcac87c521c11/tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3MB)\n","\u001b[K     |████████████████████████████████| 3.3MB 45.7MB/s \n","\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n","Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from transformers) (3.13)\n","Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n","Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n","Collecting sacremoses\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/75/ee/67241dc87f266093c533a2d4d3d69438e57d7a90abb216fa076e7d475d4a/sacremoses-0.0.45-py3-none-any.whl (895kB)\n","\u001b[K     |████████████████████████████████| 901kB 40.4MB/s \n","\u001b[?25hRequirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (4.5.0)\n","Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n","Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n","Collecting fsspec\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0e/3a/666e63625a19883ae8e1674099e631f9737bd5478c4790e5ad49c5ac5261/fsspec-2021.6.1-py3-none-any.whl (115kB)\n","\u001b[K     |████████████████████████████████| 122kB 56.1MB/s \n","\u001b[?25hRequirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.12.2)\n","Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.4)\n","Requirement already satisfied: pyarrow<4.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (3.0.0)\n","Collecting xxhash\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/4f/0a862cad26aa2ed7a7cd87178cbbfa824fc1383e472d63596a0d018374e7/xxhash-2.0.2-cp37-cp37m-manylinux2010_x86_64.whl (243kB)\n","\u001b[K     |████████████████████████████████| 245kB 59.2MB/s \n","\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.1.5)\n","Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from huggingface-hub==0.0.12->transformers) (3.7.4.3)\n","Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n","Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n","Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n","Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n","Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.5.30)\n","Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2018.9)\n","Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.1)\n","Installing collected packages: huggingface-hub, tokenizers, sacremoses, transformers, fsspec, xxhash, datasets\n","Successfully installed datasets-1.8.0 fsspec-2021.6.1 huggingface-hub-0.0.12 sacremoses-0.0.45 tokenizers-0.10.3 transformers-4.8.2 xxhash-2.0.2\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"BDh5EsI_BOXS"},"source":["## Loading Pre-Trained Model"]},{"cell_type":"code","metadata":{"id":"x9RjPCc76GjE","colab":{"base_uri":"https://localhost:8080/","height":35},"executionInfo":{"status":"ok","timestamp":1625398170258,"user_tz":-180,"elapsed":28,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"2956b1e5-7d27-47b9-f70e-dd32944408bc"},"source":["from torch import cuda\n","device = 'cuda' if cuda.is_available() else 'cpu'\n","device"],"execution_count":3,"outputs":[{"output_type":"execute_result","data":{"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"},"text/plain":["'cuda'"]},"metadata":{"tags":[]},"execution_count":3}]},{"cell_type":"markdown","metadata":{"id":"4p8rHzYVBUJU"},"source":["### Obtaining and Preparing downstream task data"]},{"cell_type":"code","metadata":{"id":"jeUi2zttmYBO","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625398170260,"user_tz":-180,"elapsed":25,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"4e2ed6c9-01ab-41c7-fd5f-05e26d75f19d"},"source":["if \"TTC4900.csv\" not in os.listdir():\n"," !wget  https://raw.githubusercontent.com/savasy/TurkishTextClassification/master/TTC4900.csv\n","else:\n","   print(\"Already there !\")"],"execution_count":4,"outputs":[{"output_type":"stream","text":["Already there !\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"XGvn9cWEtXcE","colab":{"base_uri":"https://localhost:8080/","height":204},"executionInfo":{"status":"ok","timestamp":1625398175947,"user_tz":-180,"elapsed":397,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"8ddd76d1-8416-481f-bad5-52a6d4bb0672"},"source":["import pandas as pd\n","data= pd.read_csv(\"TTC4900.csv\")\n","data=data.sample(frac=1.0, random_state=42)\n","data.head(5)"],"execution_count":6,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>category</th>\n","      <th>text</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>4657</th>\n","      <td>teknoloji</td>\n","      <td>acıların kedisi sam çatık kaşlı kedi sam in i...</td>\n","    </tr>\n","    <tr>\n","      <th>3539</th>\n","      <td>spor</td>\n","      <td>g saray a git santos van_persie den forma ala...</td>\n","    </tr>\n","    <tr>\n","      <th>907</th>\n","      <td>dunya</td>\n","      <td>endonezya da çatışmalar 14 ölü endonezya da i...</td>\n","    </tr>\n","    <tr>\n","      <th>4353</th>\n","      <td>teknoloji</td>\n","      <td>emniyetten polis logolu virüs uyarısı telefon...</td>\n","    </tr>\n","    <tr>\n","      <th>3745</th>\n","      <td>spor</td>\n","      <td>beni türk yapın cristian_baroni yıldırım dan ...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["        category                                               text\n","4657  teknoloji    acıların kedisi sam çatık kaşlı kedi sam in i...\n","3539       spor    g saray a git santos van_persie den forma ala...\n","907       dunya    endonezya da çatışmalar 14 ölü endonezya da i...\n","4353  teknoloji    emniyetten polis logolu virüs uyarısı telefon...\n","3745       spor    beni türk yapın cristian_baroni yıldırım dan ..."]},"metadata":{"tags":[]},"execution_count":6}]},{"cell_type":"code","metadata":{"id":"5jyOH3YGvyYW","executionInfo":{"status":"ok","timestamp":1625398177852,"user_tz":-180,"elapsed":4,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}}},"source":["labels=[\"teknoloji\",\"ekonomi\",\"saglik\",\"siyaset\",\"kultur\",\"spor\",\"dunya\"]\n","NUM_LABELS= len(labels)\n","id2label={i:l for i,l in enumerate(labels)}\n","label2id={l:i for i,l in enumerate(labels)}"],"execution_count":7,"outputs":[]},{"cell_type":"code","metadata":{"id":"MQ6iJXbFn86y","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625398179284,"user_tz":-180,"elapsed":4,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"51bcfc45-6c5e-4f88-a971-ac34e7b6d2bd"},"source":["label2id"],"execution_count":8,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'dunya': 6,\n"," 'ekonomi': 1,\n"," 'kultur': 4,\n"," 'saglik': 2,\n"," 'siyaset': 3,\n"," 'spor': 5,\n"," 'teknoloji': 0}"]},"metadata":{"tags":[]},"execution_count":8}]},{"cell_type":"code","metadata":{"id":"EJYWAXVovqOQ","executionInfo":{"status":"ok","timestamp":1625398182914,"user_tz":-180,"elapsed":395,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}}},"source":[""],"execution_count":8,"outputs":[]},{"cell_type":"code","metadata":{"id":"HbdFN3VZoawV","executionInfo":{"status":"ok","timestamp":1625398182915,"user_tz":-180,"elapsed":8,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}}},"source":["data[\"labels\"]=data.category.map(lambda x: label2id[x.strip()])"],"execution_count":9,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":204},"id":"KGAcx7nqwJgm","executionInfo":{"status":"ok","timestamp":1625398183337,"user_tz":-180,"elapsed":6,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"5836de30-de03-480c-8db3-2ab403ac142f"},"source":["data.head()"],"execution_count":10,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>category</th>\n","      <th>text</th>\n","      <th>labels</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>4657</th>\n","      <td>teknoloji</td>\n","      <td>acıların kedisi sam çatık kaşlı kedi sam in i...</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>3539</th>\n","      <td>spor</td>\n","      <td>g saray a git santos van_persie den forma ala...</td>\n","      <td>5</td>\n","    </tr>\n","    <tr>\n","      <th>907</th>\n","      <td>dunya</td>\n","      <td>endonezya da çatışmalar 14 ölü endonezya da i...</td>\n","      <td>6</td>\n","    </tr>\n","    <tr>\n","      <th>4353</th>\n","      <td>teknoloji</td>\n","      <td>emniyetten polis logolu virüs uyarısı telefon...</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>3745</th>\n","      <td>spor</td>\n","      <td>beni türk yapın cristian_baroni yıldırım dan ...</td>\n","      <td>5</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["        category                                               text  labels\n","4657  teknoloji    acıların kedisi sam çatık kaşlı kedi sam in i...       0\n","3539       spor    g saray a git santos van_persie den forma ala...       5\n","907       dunya    endonezya da çatışmalar 14 ölü endonezya da i...       6\n","4353  teknoloji    emniyetten polis logolu virüs uyarısı telefon...       0\n","3745       spor    beni türk yapın cristian_baroni yıldırım dan ...       5"]},"metadata":{"tags":[]},"execution_count":10}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":483},"id":"zeNYOXNBGe1B","executionInfo":{"status":"ok","timestamp":1625398187147,"user_tz":-180,"elapsed":712,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"68de8dcd-b392-4915-e5c4-5d21f5004958"},"source":["data.category.value_counts().plot(kind='pie', figsize=(8,8))"],"execution_count":11,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<matplotlib.axes._subplots.AxesSubplot at 0x7f32dc9b2b10>"]},"metadata":{"tags":[]},"execution_count":11},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAdAAAAHBCAYAAADZ1+o+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXwU9f0/8Nd7N+cGCKdcAcJpIq5QQO4AptYLwaO2trX1aGu1pmq1rc23fn823x4ar+pXq6Z+VYxntbZqaxRBkBDuS+4A4UggF4Qj97G7M+/fH7toQCCbzc5+Zmbfz8cjjyS7szOvkDDvnZnPvD/EzBBCCCFE5zhUBxBCCCGsSAqoEEIIEQIpoEIIIUQIpIAKIYQQIZACKoQQQoRACqgQQggRAimgQgghRAikgAohhBAhkAIqhBBChEAKqBBCCBECKaBCCCFECKSACiGEECGQAiqEEEKEQAqoEEIIEQIpoEIIIUQIpIAKIYQQIZACKoQQQoRACqgQQggRAimgQgghRAikgAohhBAhkAIqhBBChEAKqBBCCBECKaBCCCFECKSACiGEECGQAiqEEEKEQAqoEEIIEQIpoEIIIUQIpIAKIYQQIZACKoQQQoRACqgQQggRAimgQgghRAikgAohhBAhkAIqhFCGiBpNkOFOIrpZdQ5hPcTMqjMIIaIUETUyczfVOYQIhRyBCiEMR0Q/JKJ1RLSZiP5GRM7Tnu9LRKuJaC4RpRLRUiLaSkRLiGhoYJlXiegZIlpFRPuJ6IbA40REjxPRdiLaRkQ3Bh6fQ0SFRPRhYPlcIropkGMbEY0MLJdDRL+O9L+JsD4poEIIQxFROoAbAcxg5vEANAA3tXu+P4ACAA8xcwGAZwHkM/NFAN4E8Ey71Q0EMBPA1QByA49dD2A8gHEALgXwOBENDDw3DsCdANIB/AjAGGaeDOAlAHeH/6cV0SRGdQAhhO19E8BEAOuJCAASARwJPBcLYAmALGYuDDw2Df6iCACvA3is3bo+YGYdwM5A4QX8BfVtZtYAHCaiQgAXA6gHsJ6ZqwCAiPYBWBR4zTYAl4T1pxRRRwqoEMJoBP8R5X+d4TkfgI0ALgdQeIbnT9d22no7s7ze7nsdsv8TXSSncIUQRlsC4AYiOg8AiKg3EQ0LPMcAfgwgjYh+G3hsFYDvBb6+CUBRB+svAnAjETmJqB+AWQDWhfMHEOJM5B2YEMJQzLyTiP4bwCIicgDwAsgCUBZ4XiOi7wP4NxE1wH9tcgER/QZADYDbOtjE+/Cf9t0Cf0F+gJmriSjNmJ9ICD+5jUUIIYQIgZzCFUIIIUIgBVQIIYQIgRRQIYQQIgRSQIUQQogQSAEVQgghQiAFVAghhAiBFFAhhBAiBFJAhRBCiBBIJyIhVMpJjgOQHPjoEfhIPu1zd/ibrndEB9AIfxP1utM+f/V1Tl1TeH8IIaKTdCISwig5yS4AQ9p9pJzh+2QFydoAlAM41O7j1O9z6o4ryCWEpUgBFaKrcpL7AxgL4MLA57EA0gD0URmri5oBlADYEfjYHvi8Hzl1stMQAlJAhQheTnICgAnwT9Lcvlj2VRkrwpoBFOOroroNwHrk1B1TmkoIBaSACnEWUxeMHbamrHwmgKmBj3EI7lpkNNoLYE3gYxWArcip09RGEsJYUkCFAODOdxP8R5UZ7T4Gf3aw4nB/TeuvNJw11QNYDf9cnUUA1iKnru3cLxHCWqSAiqjlzncnA7gMwJUArgAw8PRlHjx6fM33GhqnRjqbDTUB+BzAxwA+Rk5dmeI8QnSZFFARVdz5bjeAqwIf09HBrVxTWloLX6o+MjsS2aLMTgCfwF9Qi5BT51WcR4hOkwIqbM2d746F/yjzGviPNFM68/okXd+xpqx8rBHZxJcaAHwG4CMA7yOn7oTiPEIERQqosJ3A9cxZAL4P4AZ05XYSZu+GskNaPCMhTPHEuXkALATwNoB/I6euWXEeIc5KCqiwDXe+ewKAHwC4EZ080jyX56qPbJnV0jouXOsTQWsE8CH8xfRT5NT5FOcR4hRSQIWlufPdIwH8EP6jzfON2MY1DY2Ffzp6XK6DqnUUwHsA3kJOXZHqMEIAUkCFBbnz3U4A8wDcBeBSAGTk9gb4fOsWH6qcbOQ2RKcUA3gBQD5y6upVhxHRSwqosAx3vrs/gNsB/Az+XrIRQczHtpYesnJbPrtqAvAmgOeQU7dVdRgRfaSACtNz57sz4D/avB5AnIoM/yqvOjDa6x2uYtsiKCsBPA/gPeTUeVSHEdFBCqgwJXe+Ox7AzQDuBuBWHAd3nahd8fPa+pmqc4gOHQHwEoBnkFN3WHUYYW9SQIWpuPPdLgB3APg1gEGK43zpgra2Fe9UHpYCah0tAF4G8Bhy6g6pDiPsSQqoMIVAW70sAL8E0E9xnK+J03nfxrJDI1XnEJ3mBfAagFzk1O1VHUbYixRQoZQ7390HwH0AfgE1k0sHh5mLDlbU9tT1XqqjiJBoAN4F8Gfk1O1QHUbYgxRQoURgRO0D8J+uTVIcJyh/rjm2fn5j08Wqc4guYfibM/wROXWbVIcR1uZQHcDOiCiViLaHsjwRjSeiq4xLp4Y7353kznf/Hv75I++HRYonACx2JTapziC6jABcC2ADcpLfQk7yMNWBhHWdcyYKodR4AJPgn60iKERE8J9V0A1LFaJA84PbAPwBZ5g2zAo2J8TL6Vv7IPi7V12PnORn4T+1W6s4k7AYOQKNECIaQURfENHFRPQqEd3Q7rnG05aNg7/Q3EhEm4noRiLKIaJft1tme+CINZWIdhPRawC2I4INBoLlzndfCWAzgP+DRYsnANQ6HKN9gPRjtZd4+Ed870NO8i+RkxyrOpCwDimgEUBE5wP4J4BbmXl9R8szswfAQwDeYebxzPxOBy8ZDeB5Zh7LzKaZqNid7x7vzncvhv8o+kLVebqMyPVFQvwe1TGEIXoDeApAMXKSv6M6jLAGOYVrvH7wD1q4npl3GrSNMmZeY9C6O82d7z4PwGMAfgSbvUlbnOSqubi1TXUMYZyRAN5FTvIaAL9ATt1G1YGEeUkBNV4dgIMAZgI4WUB9CBQWInIguPZ0X74moP38lKYY3BKYh/Mn8BdPW14vXJmYoKSVoIi4qQDWIif5rwD+Gzl1jR29QEQfWx0dmJQHwHUAbiaiHwQeKwUwMfD1fABnuu7SAKB7u+9LAUwAACKaAMBUfVnd+e40AIXwX+e0ZfEEgPKYmFTVGUTEOAHcC2AncpLnqw4jzEcKaAQwcxOAqwHcR0Tz4S8ys4loC4BpOPMR5OcALjg5iAj+a6i9iWgH/E0HTHEtzp3vjnfnu/8HwBYAGarzGE0nGlge46xQnUNE1BAAHyIn+V/ISR6sOowwD2mkIELmznfPAfA3AGMUR4moXx87seqW+obpqnMIJeoBPAjgeeTUme52MRFZcgQqOs2d7+7pzncvgP8oOaqKJwAsTUr0qs4glOkB4FkAq5GTbP2R5aJL5AhUdErgqPM1mPB+00hJ1PVd68rK01TnEMq1AcgG8L/IqZMdaRSSAiqC4s53xwL4E/w3nUf3mQtmbU1ZeUsSczfVUYQpLAJwK3LqqlQHEZEV3TtCEZTACNs18Dd/l78ZIueqxIQS1TGEaVwGYCtykq9VHUREluwMxTm5890/B7ARgVtohN/iJFe96gzCVPoCeB85yS8iJ9kyEySIrpFTuOKM3PnufgBeBjBPdRYz6ufzbVh6qHKS6hzClPYAuAk5dRtUBxHGkiNQ8TXufPclALZCiudZ1Tido9k/t6QQpxsDYBVykn/d4ZLC0qSAilO4892/AbAYwADVWUyNKHlnXNxe1TGEacUCeBw5ye8hJ7l7h0sLS5ICKgAA7nx3d3e++z34+9g6VeexgsVJidWqMwjT+zaAdchJltuebEgKqIA73z0GwFr4/7OLIC13JZLqDMIS0rzs/M/o7A+uUR1EhJcU0CjnzndfAX/xTFedxWoOxMYOVZ1BmB8z2m7x/rbNi9j3U7MLclKzC+SNl01IAY1i7nz3AwAKAPRUncWKfERDjzodNapzCHN7Rrtu3Sr9wrEACMDvAfwzNbtAmnDYgBTQKOTOd8e5892vA3gU8jfQJZ+7EverziDMa5M+qugp33dOn6XoOgCrU7ML5AyGxcnOM8q48909AHwC4Ieqs9jBZy5Xq+oMwpzq2LX1u56Hpp7l6QsBrErNLnBHMpMILymgUcSd7x4IYDmATNVZ7GJbfHxv1RmE+WhM1Ze1PTbAh5jYcyw2GEBRanbBnAjFEmEmBTRKBPrZrgYwTnUWO2lw0BiPf1YOIQAAzPDc4s2uOYze5wWxeDKAhanZBd81OpcIPymgUcCd754GYAWAYaqz2A5R/PrEhD2qYwjzeEGbv3aF7u7Mqdl4AH9PzS6416hMwhhSQG3One+eD2AJgD6qs9jVoiTXcdUZhDls0UcUPeb73umDhoJBAJ5OzS54XG5zsQ4poDbmznf/DMC/ACSqzmJnaxISElRnEOrVc+L2Gzw5U7q4ml8DeD01u+Bc106FSUgBtSl3vvs+AH+DtOUzXFWMc4TqDEItjenI5W2P9fUiJi4Mq7sJwHup2QXhWJcwkBRQGwoUz7+ozhEtmKjf/tiYg6pzCDWY4b3N+0B1FfqEcwKG+QD+IUXU3KSA2owUTzU+c7mkgEap/9Pmrl6uj7vIgFVLETU5KaA2IsVTnc9diTI3aBTarqcWPey7aZaBm5AiamJSQG1CiqdaJXGxMn9qlGnkhJ3Xe/5ncgQ2JUXUpKSA2oAUT/XaiEbVO6hOdQ4RGRpTzeVtj/byIDY+QpuUImpCUkAtzp3v/iWkeKpHREWJiXtVxxDGY4b3p95fV1ag38AIb/pkEZVbXExCCqiFufPdtwB4SnUO4bc4ydWgOoMw3gLtitWf699Q1RJzPoAF0mzBHKSAWlRgIuyXVOcQX9mUEJ+sOoMwVrE+dMUffDcbOWgoGDfBPxWhUEwKqAW5892TALwHIEZ1FvGVEw7HaA3QVOcQxmjihOJrPX+YpDpHwG+kd656UkAtxp3vHgWgAECS6iziNETdtsbHlaiOIcJPZzp6hSe3RxvizNS28SmZxUUtKaAW4s53nwdgIYBgpkkSCixKch1WnUGEFzN8d3jvKz/E5w1WneU0BOA1mU9UHSmgFuHOdycB+AjASNVZxNmtSEyUEZI285r2rVWL9UnjVec4i3gAH6RmF3Rm+jQRJlJALcCd746B/5rnxaqziHM7FBsjc67ayG49ZeXvfbepHjTUkWQAn6RmFwxRHSTaSAG1hmcAXKE6hOiYRjS4yumsUp1DdF0zx++6xvPHCapzBGkwgILU7AIZGxFBUkBNzp3v/gmAn6vOIYK3JCmxVHUG0TU60/ErPLndWhFvpbl03QBeUR0imkgBNTF3vnsKgOdU5xCds9Tl8qjOIELHDO0u7z2lB7l/iuosIfhuanbBA6pDRAspoCblznf3B/BP+AcJCAvZER/XT3UGEbq3tcwVC/UpVjl1eyYPp2YXfEt1iGggBdSE3PnuWAD/gP+6hrCYZqIxLUTNqnOIzturD1r1O99PZ6vO0UVOAH9PzS4YrjqI3UkBNaenAGSoDiFCRBSzJjFBGipYTDPH7Z7n+bNZb1fprN4A3k/NLnCpDmJnUkBNxp3vvhVAluocomsWu1y1qjOI4OmME1d5HnG1IN5OBWccpF+2oaSAmkigx22e6hyi69Yl2mpHbGvM0O723rO/lAfa8T7K76dmF9yvOoRdSQE1CXe+uxuAtyGDhmzhiNM5igFWnUN07B/a7BUF+tSJqnMYKDc1u8DOP58yUkDN438BjFIdQoQHE/XaHRe7X3UOcW4H9P6rH/DdYfVBQx2JBfCWXA8NPymgJuDOd38bwI9V5xDhtdjlko5EJtbCcSVzPY9cpDpHhIyBf3CiCCMpoIq5892DAbyoOocIv+UuKzWxiS46o3au5+G4ZiREU+u7n6VmF1yjOoSdSAFVyJ3vJgD58A85FzazLy5W7uM1IWbo93mz9u7nQdHY+P+l1OyCgapD2IUUULV+BeCbqkMIY3iJhp9wOI6rziFO9b4+s+hDfcYk1TkU6Qvg1dTsAlIdxA6kgCriznePB/Bn1TmEsZa5EveqziC+Uqaft+Z+7112HzTUkcsA3Ks6hB1IAVXAne9OAPAmgDjVWYSxFie5WlRnEH6tHLvvKs8jF6rOYRK5qdkF8m/RRVJA1XgQwAWqQwjjbYmP66k6gwCYUTfP82dnExK7qc5iEvHwXw+VGtAF8o8XYe589wUAfqs6h4iMeofjfC/gVZ0jmjFD/7X3zj0lnJKqOovJTIHMNdwlUkAjKDDq9kX4b2wW0YAoYVNC/B7VMaLZv/VpRf/UZ12sOodJPZyaXTBIdQirkgIaWbcDmKE6hIisRUmuo6ozRKtDet+193p/MUt1DhPrAeBZ1SGsSgpohLjz3QMAPKo6h4i81YkJ0t9YgTaO2X+F59ELAJJbNs7t+tTsgvmqQ1iRFNDIeRqADCiJQhUxMTKxcYQxo36+50/UhMTuqrNYxF9TswtkgFUnSQGNAHe++0oAN6rOIdTQifqXxcSUq84RLZjBv/Xdvms3D5U3LsEbAuCPqkNYjRRQg7nz3S4Az6vOIdRakpRYpjpDtPhYn1L4rnbJZNU5LOju1OyCaO3QFBIpoMZ7AECq6hBCraUul6Y6QzSo4D7rsrz3RHunoVA5ATyjOoSVSAE1kDvfPRDAr1XnEOrtjovtrzqD3bVxzIHL2x5Nk0FDXTItNbvgBiNWTEQ9ieiuIJZLJaLtYdpmKRH17WCZVYHPg4jovc6sXwqosf4AIJqmSxJn0Uo0upGoQXUOu2JGw7WeP+qNcPVQncUGclOzC4y4V70ngA4LaKQx8/TA50pm7tSbBymgBnHnuy8EcJvqHMIkiBwrXYklqmPYETP4Qd+PdxTzsJGqs9jESBhT6HIBjCSizUT0OAAQ0W+IaD0RbSWi/zn9BUQ0goi+IKKLiehWIvoXES0kohIieqzdct8nom1EtJ2Izni7IBHdH3h+OxH9st3jjYHPnT7ylQJqnMfgv6YgBABgsSuxXnUGO1qkT1z+lnbpVNU5bOb/pWYXhPu2u2wA+5h5PDP/hoguAzAawGQA4wFMJKIvm14Q0fkA/gngVmZeH3h4PPx3NLgB3EhEQ4hoEPz32GcGnr+YiK5tv2Eimgj/Ac0UAFMB3E5E3+jqDyQF1ADufPelAK5UnUOYy4aEBDm9GGbV3Gv9nd77MlTnsKE+8E96YaTLAh9fANgEIA3+ggoA/QB8COAmZt7S7jVLmLmOmVsB7AQwDMDFAJYxcw0z++Cf6er07lMzAbzPzE3M3AjgXwC6/HcjBTTM3PluB4DHVecQ5nPM6RilA7rqHHbhYWfZZW2PjmE4ZD9mjLtTswtSDVw/AXgkcEQ6nplHMfPLgefqAByEv/C119buaw1AjIH5OiR/eOH3I/hPIwhxKqIe2+PjZILtMGBG43WeP3jr0S1ZdRYbiwfwcBjX1wCgfWeoTwH8mIi6AQARDSai8wLPeQBcB+BmIvpBB+tdB2A2EfUlIieA7wMoPG2ZIgDXEpGLiJIC6y7q2o8jBTSs3PnueEg3D3EOi12uatUZ7OAh363bd/DwUapzRIHvpWYXTAzHipj5GICVgUE8jzPzIgBvAVhNRNsAvId2BZaZmwBcDeA+Ijprr15mroL/+urnALYA2MjMH562zCYAr8JfbNcCeImZvzj5dKg/EzGH/FpxGne+OwvAX1XnEOY10uNZ9UFF9XTVOaxsifaNwp94fyPNEiLnP6W5c23ZbJ6I+gDYxMzDQnm9HIGGiTvfHQeZKFt0oCw2dojqDFZ2hHtu+Kn3VzJoKLLmpWYX2O6yVGD07moAT4S6Dimg4XMb/A2ZhTgrH9GQI07nEdU5rMjLzoPfantslAwaUuIh1QHCLdA4YQwzhzwfqvwhhsEffnqBE8xy9CmCstSVuF91BqthRtO3PTltdegmUwKqcW1qdsFFqkOYjRTQMPjOCv7R609ovqvX6qscOkvTcHFOS5JcbR0vJdr7g+9HW7fyyNEdLykMQgD+S3UIs5FBRF1UnJZOAHYASAcAnwNlH0yjg/+c4ZiqOcmIfpLC4rrp+o7VZeVjVeewimXaRYW3erNl0JB6GoDzS3Pn7lMdxCzkCLTrrkWgeAJAjI5hN6zkjDce147c8plWGOvjVoXZhAk1Eo1uI8jfRRBqOHnjj70PnH4zvVDDCeA3qkOYiRyBdlFxWvoa+PsrnpFOqFnmpp2vfssxsTXOf8OwEM9VH9k6q6VVrimdg5edhy5ue75bLbr3Up1FfKkNQGpp7ly5nxlyBNolxWnpGThH8QQAB6Nf5laenf+k5v3l+9qybi1cG6F4wsQWJ7lOqM5gZsxo/o7n981SPE0nHsDdqkOYhRTQrvlFsAsS0Gv6Lp7z8tOa87/e0QqTG7nGyGDC3NYmJiSqzmBmj/h+sHkzjzpfdQ5xRj9NzS6IUx3CDKSAhqg4LX0A/P0UO4WA7t/Yz7NffFbrlvOGr7BfLVcaEE+YXLXTKXNXnsUK7cLCF7WrpVuTeZ0HoFMTT9uVFNDQ3Q4g5FG2BCRecAiz//qC1veRBb4Vg45xWRizCZNjoj57Y2MPqM5hNke5x6abvdkyaMj8jJhw23JkEFEIitPSnQBKAaSEa50MaBV9sObZ+c7+BwaQNMmOAnedqF3x89p6KRYBPnaUT257LvE4kvuoziKCMq40d+5W1SFUkiPQ0MxHGIsnABDgTDmGGbkLtJHPPu9bk3aIi8O5fmE+y1yJ8u41gBkt3/U81CjF01Ki/ihUjkBDUJyWvhjApUZv50QSNr4w1xG7eaRDbnewoTid920sOyTXQgHker+3Mk+bP0N1DtEpjQAGl+bOrVcdRBU5Au2k4rT0MQC+GYlt9WrCxN+9q1/08lO+LdN36hsjsU0ROR7CiDqHI+pva1qjpy+X4mlJ3QDcrDqESlJAO+/n8PeFjJjurRj3yw/1iflP+nZmbtbXQk4b2AMRFboSSlTHUOk4d998k+d3MuLWun6uOoBKcgq3E4rT0uMAVANQenN3WwxK3pnlqCmYTFOZSN4EWdicpubCZ48cjco+rz52VE5pey7uGJL7qs4iuiSjNHfuCtUhVJCdb+dcCcXFEwDifRh981J9+huPaQe/U6QVOTX2qs4kQrM5IT5ZdQYVmNH6A8+DtVI8beEHqgOoIgW0c0z1hxKrI/U7K/yN62/+TFsujeutp9bhGOMDfKpzRNqTvu9sWMfpF6jOIcLiO6nZBTGqQ6ggBTRIxWnp3QDMU53jTJyMwVev51mvP6HV3/GxtizBw42qM4kgEbk2J8TvUR0jktbr5y//q3ad3P9qH30BfEt1CBWkgAbvWgCm7l/qYJz3zS08J9C4vlAa11vDIpfrqOoMkVLLSVu+73lwmuocIuxMdXYuUqSABu/7qgMEK9C4fvbLT2vO7He1ZdK43txWuRKiYuJ1jR1Vl7U9NsiHmKj4eaPMtanZBaY+wDCCFNAgFKel9wVwmeocnUVA9wn7eM6Lz2rdfv+Gr7BvHVepziS+7lBMTKrqDEZjRttN3v86fgS9+qnOIgxh2ktcRpICGpwbAFj2IjkBiWMPYfZzz2t9HlngK5LG9eaiEw0sj3Haelae/9WuX7dGHztWdQ5hKMucpQsXKaDBscX5fQLiRlYj46kXtZS/vOhblVrN+1RnEn5LXa5S1RmMskkfVfS074YM1TmE4a5MzS7oqTpEJEkB7UBxWvpgALYaMRhoXD/90QXaCGlcbw5LkhJteStLHbu2ftfz0FTVOURExCOEOZKtTApox+Yiwq37IoUA6l+HqX94Q0vPe9a3Yfw+PaqnJlKpOC7OdtcGNabqy9oeGyCDhqLK1aoDRJIU0I5dpTpAJPRuxKSTjeunFUvj+khrIRrTTNSkOke4MMNzize75jB6n6c6i4ioS1OzC6LmDZMU0HMI9L6NyMwrZtG9FePu+0Aa10cckXNVon0ayz+vzV+7Qne7VecQEdcDQNTMrCMF9NxmwT88O+okenDBnZ/oU15/Qts7d52+iph11ZnsbnGSyxaNL7boI4oe931PBg1Fr6g4awdIAe1I1PwhnE28D6NvWSKN6yNhfUK85d+s1XPi9hs8OVNU5xBKRc1+U6YzO4fitPRiAGmqc5iJRqj4ZBLte3uOY7I3hhJU57EV5rqtpYd6kEUHrWlMR2a2PaNXoc8A1VmEcsNKc+ceVB3CaHIEehbFaenDIcXza05rXF8ojevDiCh5Z1ycJe/NZYb3Nu8D1VI8RcCVqgNEghTQs4ua0xChCDSun53/pOa99wNtmTSuD4/PkhIt2W7x/7S5q5fr4y5SnUOYRlTsP6WAnl1UvIPqKgJ6zSjmOYHG9YXSuL5rCl2Jlvs/uV1PLXrYd9Ms1TmEqXwzNbsgXnUIo8k10DMoTkt3AjgBoLvqLFbDQMvOIVj33DznmKPJNFB1HquJZS7bVHpomOocwWrgxB0T2/JGeRBr+52l6LSM0ty5K1SHMJLl3u1GiBtSPEPSvnH9w6/6igYeY9sPJAgnL9Gwo06HJY7iNaaaK9pye0vxFGcxXXUAo0kBPTPb/+KNRkDcqCpkPP2iNvjJ//OtlMb1wVvmSjygOkNHmOH9qffXlRXoJ2cZxNnYfj8qBfTMoqaThtEIcA45ihmPLtBGPPuCb8350ri+Q5+5XM2qM3RkgXbF6s/1b4xTnUOYmu0LqFwDPYPitPQDAFJV57Cr492w4YWrHHFbRjpk1OYZdNf0basOlpu2DV6xPnTFlZ5cW81QJAwzpjR3rm1aVJ7OspNEG6U4LX0gpHgaqncjJj34ro6GRH3zS5c7tNXpjomqM5lJg4PGeIC2OP/0UKbSxAnF13r+MEl1DmEZMwDYtoDKKdyvk9O3EdK9BePv+0Cf+OqTvh2Zm/V10rg+gCh+gwkby+tMRy/3PJrchjjpQCWCZevTuFJAv87Wv3Azcnkw9s5P9MnSuP4ri1yuY6oztMcM3x3e+8rLud8g1VmEpdj6gEQK6NdJAVXky8b1j2tlNxTpK6K5cSzarcEAACAASURBVP2axARTnb59TfvWqsX6pPGqcwjLSU/NLuilOoRRpIC2E5j/8xuqc0S7WA3Dv7tCn/nG49qRHy3Rlsf6uFV1pkirjHGOVJ3hpN16ysrf+26TTkMiFARgsuoQRpECeqrzAcSpDiH8nIzB89b5G9f/LMoa1zNRvwOxMWWqczRz/K5rPH+coDqHsDTTjijvKimgpxqrOoD4OgfjvEu/alxfmNTCdaozRcJnLle5yu3rTMev8OR2a0V8osocwvJsu1+VAnqqC1UHEGcXaFw/+5WnNfrtu9qy5CY+qjqTkT53JWqqts0M7S7vPaUHuX+KqgzCNmy7X5UCeirbvlOyEwJ6TNzHc158RnP9/k2tsE8dW3IKsI7siYtV1ibvbS1zxUJ9ipy6FeGQnppdYMlJ4jsiBfRUtn2nZEcEuMYe5NnP27RxfRvRyHoHRfx09V590Krf+X46O9LbFbaVBGC46hBGkAIaUJyWnghghOocovNs27ieyLEiMTGiDRWaOW73PM+f5XYVEW62PLsnBfQr6ZB/D0uzY+P6xUmupkhtS2ecuMrziKsF8a5IbVNEDVue3ZNeuF+x5TukaEQA9a/F1D++oVm+cf2mhPiIzEvLDO1u7z37S3mg9CUWRrDl/lUK6Fds+Q4p2p1sXF+fqG956XKHtibdYamBMccdjjEaoDkBp5Hb+Yc2e0WBPlWuewqj2HL/KqcsvzJKdQBhnB4tGHf/B/qEV5/07bhki4Ua1xN12xYfZ+h10P36gNUP+O6Q4imMZMv9qxTQrwxRHUAYz+XB2J9/rE9+/Qmt5CqLNK5flOQ6YtS6Wziu5GrPw5Y8vS0sJcmOPXGlgH5FCmgUifdhzK0WaVy/IjHRkEstOqN2rufhuGYkJBmxfiFOY7t9rBRQfNlEvr/qHCLyvmxc/4R22KyN6w/GxgwN9zqZod/nzdq7nwcNC/e6hTgLKaA2NRj+WQNElHLqSPmycf0nWmG8hyN2+0hHNKKUKqczrN2W3tdnFn2oz5gUznUK0YHoLKBE9CQR2XIYcoDtfrEiNA7GeZdu5tn5T2pt93yoLTNL4/qlSYml4VpXmX7emvu9d8mgIRFpttvPBnsEWgzgRSJaS0R3ElGykaEUkIbZ4hQOoPfMnTznlac1euAf6hvXL3G5wnKNtpVj917lecSWtxQI07PdfjaoAsrMLzHzDAA3A0gFsJWI3iKiS4wMF0G2e2ckwoOAHpP2qm9cvzM+rm9X18GMurmeh2ObkNgtHJmE6CTb7WeDvgZKRE4AaYGPowC2ALifiP5uULZIst0vVoSX6sb1TURjWoiaQ309M/T7vT/fs48Hy6AhoYrt9rPBXgN9CsAuAFcBeJiZJzLzo8w8D8A3jAwYIbY7tSCMcXrj+mGHI9S4nihmbUJCyA0V/q1PK3pfz7g4nJGE6CTb7Wc7LKBERACOAxjPzHcw87rTFplsSLLI6qM6gLCWk43rH3tFG/HMC741Y8p5l9HbXJSUeCKU1x3S+6691/uLWeHOI0QnJaRmF9hqooIOCyj7W559l/nMw/qZzTFKsYt6qA4grIkAGlCLqX96XUvLe9a34aL9+jajtrUusfMND9o4Zv8VnkcvAEhu0xJmYKt9bbDXQDcRkZ1P/9htVLFQoHcjJv33O7r7pad9m6cW65vCvf4jTucoBoLu4cuM+vmeP1ETEiMyo4sQQbDVvjbYAjoFwGoi2kdEW4loGxFtNTJYhNnqXZFQq0cLxp9sXD/H37g+LOtlol6742IPBLUsgx/w/WzXbh46PCwbFyI8bLWvDbbH5uWGplDPVr9UYQ4uD8be9bGOnyzCnrdnO45+cjFNZaIudf/6zOWqSPPUjehouY/1Kcv/oc2RZgnCbKLvCJSZywD0BDAv8NEz8JjlFaelJ8HguRZFdGvfuP7bK/QVDp19oa6r0JXY4bXMCu6zLst7jwwaEmZkq4OVYG9juRfAmwDOC3y8QUR3Gxksgmz1CxXmFath+I1F+sw3H9eqf7hEWx7j47bOrmNfXOzgcz3fxjEHLm97NE0GDQmTstX+NthTuD8BMOXkSFwiehTAagDPGhUsgmx1SkGYn1NHyvx1nHL1eu3wknG067VvOia1xVFQI2y9QOoJh+N4L13vffpzzGi41vNHvREuW+2khK3Yan8b7PUYAqC1+16DfWYvkZ2NUMLB6P+trxrXF7pag7gljIiWuRK/1ryBGfyg78c7innYSEPCChEettrfBnsEugDAWiJ6P/D9tQBeNiZSxMkQf6FUoHH97Bk7tfqNo2jZ365yXFiXRGftfftZkqvpusZTb8tepE8qfEu7dI7RWYXoIlvtb4MqoMz8FyJaBmBm4KHbmPmLk88TUQ6ARmZ+IuwJjRfsmwghDHWycf3EZ7TmHcOo8Lm5jjHHkmng6cttiY875fRtFfdef6f3lzJoSFiBrfa3Qf0wRNQbQGng4+RjscwclimWFJNJxYWpEOC6sIxnP/+85tk3EEXPzHemVvemLxtx1zkco72ANxaI9XBM6WVtj45hOOTvWFiBrf5Og+5EBKAGwB4AJYGvjxJRKxF9AeD8kwsS0TIimhT4ui8RlQa+vpWI/kVEC4mohIgeCzz+YyJ6ut3rbw80rwcRfUBEG4loBxH9rOs/7hnJLSzClE42rv/fv2kDn3ipXeN6osRNCfF7mNF4ned/fA1IstXADGFrttrfBltAFwO4ipn7MnMfAPcA8MHfYEEDEGybv/EAbgTgBnAjEQ0B8C6AeUQUG1jmNgCvBL7+MTNPBDAJwD1EZETTd1u9IxL2Q0DM0BrMeOwVbfgzL/jWjK7g3YuSXDUP+W7dvoOHj1KdT4hOsNX+Ntjz0VOZ+fZ238cC8DFzIRHFAPh3kOtZcrL5PBHtBDCMmQ8R0VIAVxNRMYBYZj7ZkPseIrou8PUQAKMBHAtyW8Gyy2hiYXMEOAbUYup/v5tc8cW4K6pHDZg7+DKiVapzCRGs4+CjqjOEU7AFtIqIfgvg5OTZmQCaApNs66ct68NX7zISTnuu/Y3jWrvtvwTgd/DPOboAAIhoDoBLAUxj5ubAIKbT1xcOp+cXwpSaXP3LdqbferCh25CpbQ1vpKBhuza8x/gM1bmECNZw4IuOl7KOYAvoDwD8HsAH8M8GsQdAC4Ak+E+5vgfgb4FlSwFMBLAOwA3BrJyZ1wZO504AcFHg4WQAJwLFMw3A1CCzdpYUUGFq9d2GlOxMv7Wm2dV/CoiGAQBrxwZvOPbp8H4JQ1b1iOszXXVGIYJkq/1tsLexHAVwNxEltetG9CCADQCOAFjfbvEnALwbGPRT0Iks78I/affJSYMXArgzcFp3N4A1nVhXZ9jqFyrs40TyqJ3FaTc3tCb0ngyi0ScfZ735GKAPB4BFla+Ov3boPbtjHLHnn31NQpiGrfa3wd7GMh3+06zdAAwlonEABjPzmNOXZeZd+OooEgD+O/D4qwBebbfc1ae9dCaAp9o93wbgymDydZHW8SJCRM7R3mO37Dr/Jp8nPnnimZ7XvPv2AegDABr7XJ9WLHBdlXL7CSLqFdGgQnSerfa3wZ7CfQr+Ebf/BgBm3kJEYblxm4h6wn+6dwszLwnHOjvJo2CbQnzN4fMmbtg9+sZYX2zSuHMtp3tKWtp/3+g7MWR1zb83Tus3f3xgXIIQZmWr/W3QXSECo2XbPxSWdxLMXAvga0eyEVSvcNsiyjHAlQNnrNs78vruWkzCpGBeo2tVXzvSPNS0a+KAxOGFI7pfJHOACjOz1f422AJ6KHAalwP3a94LoNi4WBFlq1+osAYGaQeHZK49kHp1P90ZNyXo17HmAbed8Q3n+qOfzO6XkLK6e2zvaeFLKkRY2Wp/G+xNrXcCyAIwGEAF/A0R7jIqVIR1PAOGEGGik8O7P3Vu0bJZT5fvG3n9dN0ZN7rjV7V7va+8BOe4nevTilcv8uneki4HFcIYttrfBnsEej4z39T+ASKaAWBl+CNFnK3eEQlz0hwxrftGXLOufPDsUSBnyPdu6p6Sc96IrrE3aVHlq3FXDv5pbWB8gRBmYqv9bbBHoGeaONsOk2kjfVdxG2x2YVuYh88Z31B8/k3LCjOeaihPyZwFcg7qyvp0X1l8R8s0eI8PW1Pz0V5mttUtA8IWbFVAz3kESkTTAEwH0I+I7m/3VA/YqylwHYB+qkMI+/DGuGp3j/ne5iP9JoyDv6tWWLBePyKY5Q427Zw0MHF4YWr3C2VQkTCTqDqFGwf/vZ8xOHUi1HoE2WXIIuohBVSEQVtcj5ri83+443jvCyaGs3ACgK4dPwTwkI6X9Ft7tGB234TBa7rF9jKqi5cQnRU9R6DMXAigkIheZeayCGVSwVbvikTktST0rixOu7mkNnnU5HAXzpN0796D8E+qELRPKxZceO3Qe/Y5HTEjjcgkRCfZal8b7CCiZiJ6HMBYtBsByMyZhqSKPFu9KxKR077BO4i6dH2zI5pnX6evafrY221R5atHrxj8kzoiknlDhWq22tcGW0DfBPAOgKvhv6XlFvgn1baLI6oDCGtp6DZk7470Ww43uwZMPdng3Wis1fQP5XX13mOp645+vH5y36smEpGt5mMUllKXkpthqwGbwRbQPsz8MhHd2+607voOX2Udh1QHENZQmzyyeGfazfWtCX0mgyhik1kzt9UDvpC3V9q4/eIBicMLh3W7QAYVCVVst58NtoB6A5+riGgugEoAvY2JpITtfrEivAIN3r2e+OSg2u2Fm+4t3Qv/dH8hW1Pzn1l94wetTYrtGXTnIyHCyHb72WAL6J8C109+Bf/9nz0A/NKwVJFnu1+sCI/D/SZu3D2m4wbvRtM8e8Jx7YgWViy44Nphd+93UkxQt8MIEUa2288GW0C/A2AFM28HcAkR9YZ/3s//GJYssmz3ixWhCzR4X7935PXdtJiEM04pFmm6r6JHONbjY0/3xRX5Ry8f/ON6IgrLOoUIku32s8EW0IsCs6YAAJj5OBF9w6BMKpSrDiDUY5B2KCVz7f7hV/fTnXGTVec5iVnXwM2d6pl7LnXeo8PXH1247uK+V1xMp02xJISBbLefDbaAOoioFzOfAIDAEWjQU6FZwGH4r/PGqg4iIk8nh7d02JVry4ZeNoQdMdNV5zkda9X7EOYp/w40bp080DV82ZCktDnhXK8Q5xC1R6BPAlhNRP8IfP8dAH82JlLkpe8q1ovT0isApKrOIiJHc8S07h9+zdrywbNHscM5U3Wes9E8JdUwYM7cVUc+nH31kEHrkmJ6mOZoW9hadBZQZn6NiDYAONk44Xpm3mlcLCUOQQpoVPA54xtLRn17Q9WAaReAHKa/rUP3HjCq7zR9Wv5y2jXD7j7gpJjhBm1DiJOis4ACQKBg2q1otndQdQBhrECD9y1H+k24yKh2e0ZgvdawRg1e9vT4rPL1mssG3dpARN07foUQITmWkpvRojpEuNnpOmZX7VYdQBijLbZ7za60H+441nvsRBCZ/oizPdYbDgN6ipHbqPUcGbnx2KI1E/tcNkUGFQmD7FIdwAhSQL+yXXUAEV4t8b2ritNvLqlNHnWxlY4429M8+w4ACKmFX2fsa9g8dUDi8MKUpDGWeoMhLMOW+1cpoF/ZoTqACI+mxP5lOy+45WBDt6FTQDRQdZ6u0L0lbZHa1soj72fMG3LXeldM94sjtU0RNWy5f5UC+pW9AFrRbrYZYS0N3VL27Ui/9XCza8CUSDV4N5ruO9w3gptzLKx4ecw1Q+8uc5LTFv9+wjRseQQqMzMEpO8q1mHT8/R2V5s8snjVlP9Zs35i9ojmpIHTQWTUqNWIYva1Ap6w375yLl69LXlJ5eteZm6M5HaF7ckRaBTYDmC86hAiOMd6X7C1+PybPJ74nkoavBtN9x3cDSDiPXhPeA6P2nRs8ZqJfS+bGultC1uqScnNsOWUkVJAT2XLd0l2c7jfhE27x3zPqbrBu9F0z57ajpcyxt6GL6YOdI0oHOQaJYOKRFfZdr8qBfRUtjxPbwcMcNXA6etKRn67uxaT0KVpvaxC8x5KVLn9osP/zJg/JGtDYkw3Wx7hi4ix7X5VCuipbPtOyaoYpB9KuWTN/uHz+unOuOiax5IbIjZh91k4Fla8POqaob846CDnUMVZhHXZdr8qBfRUpQDqACQrzhH1/A3er1hTNvRyUzZ4N5qu1RwAoLy9nkdv7flZ5Rs13xp0cxMRJanOIyxps+oARpFRuO2k7ypmAGtU54hmmiOmtWTk9csLM54+Upo6N4MdMamqM6mgeUoqVGc46YSnevQXx5dsVZ1DWFIbgE2qQxhFjkC/bhWAy1WHiDY+Z3xjychvb6waOC0d5JilOo9qune/6ginKKnfOG1g4ojCga4RMqhIdMaGlNwMj+oQRpEC+nUrVQeIJoEG75uP9Jswzmp9ao3E2jHTdVAqOvzezHlDsjYmxiRNVJ1FWMYq1QGMJAX069YC0ADY4mZ8swo0eN95rPfYCVbtU2sU1ltOANoI1TlOx2DnwoqXRlwz9BeHHOQcojqPsARbH5DINdDTpO8qbgSwTXUOu2qJ7121afwvC1dOf6TbsT4XzoZMofU1unf/XgCmnBXFo7f2WlL1ZjMzN6vOIixBjkCj0EpIR6Kwak487+CO9FvLGroPnSKnas9N85aYujgdb6s6f8uJZavG974k6kZHi04pScnNqFEdwkhSQM9sFYAs1SHsINDgvbrZNWAqiORewiDovkrT30a1u27d9AGJwwsHJKbKmyFxNrY++gSkgJ6Nrc/bR0JtjxHFO9NvqWtN6DMFRCNV57EKZt0Hbj1fdY5gLK9+d+b8oVmbEpxJUdEZSnSa7QuoXAM9g/RdxWUATHMfnpUc65W+dcW0P2/YNOFX6a2JfaeCyJTX8syKfRV7ACht4RcsBjsXlr8yTGetXHUWYUq2PxCRI9Cz+wzALapDWMXhfhM27hlzo9Mb202uHXeB5i2x1DWjNr25z9Kqt3d9c+BNLURkicIvIqIyJTfDti38TpICenYfQwroOTHAVQOmrS8Z9e0kLSZR7g0MA91bGqc6Q2cda6tI23qicOW43nNmqM4iTOMT1QEiQQro2S2C3A96RoEG72v3D5/XV3fGTVadx05Yr0tVnSEUu+rWzhiYOGL5eYlDo76LlAAQJQVUroGeRfqu4loAq1XnMBOdHN4Dw65asWzW02V7R317mu6MG606k53oWm0FwKbrQBSsZdV/n96mNdu2cbgImhfAYtUhIkEK6Ll9rDqAGfgbvF+3vDDj6cMHhs+dyY4Y5bOE2JHu3VumOkNXMDjmk/KXU3TWK1VnEUqtTMnNqFcdIhKkgJ5bVBdQnzO+cdeYHxQWZjxVf2jIpbPY4UxRncnONO9en+oMXdWmN/ddVv12LTO3qs4ilImK07eAFNBzSt9VvAVReDuLNyaxbnv6bYXLZz7prRw0YzbIcZ7qTNGAfTW2+HeuaS2/YHtt0QbVOYQyUXPgIYOIOvYJgJ+qDhEJntjuR4vTbtp+rPeFE6TdXmQxexoBr22uKe+sXT1zQOLw5f0ShsigouhyKCU3Y7vqEJEiR6Ads/3piNb4XlWbxt1buGL6I65jfdxzQNRDdaZoo3vLSmCzEd+fV/19WpvWskV1DhFRtt9ftidHoB1bBKAFFukO0xn+Bu+3lDV0HyYN3hXTPHvqVGcIN4Yeu7Di5UHzhtxV5SCHZUcXi075QHWASJIj0A4Epjf7j+oc4dSQNHjf2osfXLlm8kODG3qkZoDIcjfv243uK7fltG6tWlO/wup3jjNzm+oswnBHESW3r5wkR6DBeRvAd1WH6KraHsN3FaffUtuS0FcavJsIM+vgplGqcxjlSOvBsTtqVxZd2GtmhuoswlDvpeRmWH4keWdIAQ3OxwBqAfRUHSQUx3qlby1O+6HHE99zkuos4utYO7wPgG0GEJ3JjtqVGQMShxf1TRgsRdS+3lIdINLkFG4Q0ncVewD8S3WOzjrS7xubimY8umXLuF9cJMXTvDRPSbXqDJGwtOqtqR6tdavqHMIQhwCsUB0i0uQINHhvAfix6hAdOa3Bu8zTaAG670BUTPnG0GM/qXh5wLwhP692kGOA6jwirP6ekpvBqkNEmhTQ4H0OoBqAKf/j+xu8z1mzf/h8afBuMawdH6o6Q6S0ao3nLa/+x7bZA77bm2Twmp28rTqACsQcdW8aQlaclv40gHtV52hPJ4evbOjla0qHXT6YHbHSo9ZiWG+qaav7Wz/VOSLN3WtW0QU9p8n1UHvYlZKbka46hApyBNo5b8MkBVRzxLTuHz5vXfngS0awwzlTdR4RGs27dz+AqCug204sz+ifmFrUJ36gFFHri8qjT0AGEXVK+q7itQD2qMygOeKado35fmFhxlN10uDd+nTP3qhtur608o0pHr01atq+2RQDeEN1CFWkgHZenoqNBhq8LyvM+Etb5aCZs0GO/ipyiPDSteo+qjOookOPW1j+Sl9m/YjqLCJki1JyM/arDqGKFNDOWwCgOVIb88R2P7rlwjuXFc14nI70nzQHRL0jtW1hLGZfG7jN1vd/dqRFaxiw/PB71czsVZ1FhOR51QFUkgLaSem7imsB/N3o7fgbvN/jb/DeVxq825HuK98DIF51DtWqWw5ctLt+/WrVOUSnlQH4SHUIlWQQUWieg0H3hDYn9ju0I/3WUmnwbn+6Z89x1RnMYsvxz2f1Txha1Ct+gAwqso4XU3IzdNUhVJIj0BCk7yreBGBdONfZkDR439pJD65cM/n3A6XBe3TQfQcTVGcwk88q35js1dt2qM4hguIB8JLqEKrJEWjongfQ5YYF7Rq8T5YG79GF9foRqjOYiQ4tfmHFK72vTrmjhsgRdbf2WMw/U3Izon7wlxTQ0L0D4EkAIY2iPNYrfduutB+2tsX3vDi8sYQV6NqxMgDDVOcwm2Zf/cCiw//cktH/hp5EFKs6jzirqB48dJKcwg1R+q7iVvhH5HbKkb7jNxVNf3TzlnG/cEvxjF66p6RcdQazqmrZP66kfqMMKjKvrSm5GVHXOP5M5Ai0a54HcB8AZ0cLVg6Yur5k1A0uafAuAEDz7o/qwRcd+eL4klnnJQxd0TP+POmyZT5/VR3ALOQItAvSdxUfAPCPsz3PIP3Q4Dmrl2X8ZfeutB9drMUkjo1gPGFirB015aQEZrK46rVJXt1TrDqHOEUVgNdUhzALOQLtulwA32v/wGkN3qcpyiVMivXWOsA3SnUOs9NZS1hY8Ury1Sl3HCWivqrzCADA0ym5GW2qQ5iFHIF2Ufqu4i0AFgKATjFte0dcu7ww4+nqA8Ovnimzo4gz0b0HSgBExRygXdXsqxu04si/ypnZpzqLQC2AF1SHMBM5Ag0DzRHzcMnIGxIrB81IAzlmqc4jzE3zljSpzmAllc17x+9t+GL56B4T5P+WWi+k5GY0qA5hJnIEGgYX7txWVDk4wyEN3kUwdF+FtGXspE3HFs+q89SsVJ0jijUB+IvqEGYjBTR8/qA6gDA/Zl0Dt4xRncOKFle+NsGne3apzhGlXkjJzTiqOoTZSAENk6y8zM8ArFKdQ5gba1UlAJJU57AijX2JCysWdGNm6SEcWS0AHlcdwoykgIaXHIWKc9I8JYdVZ7CyJl9tyqojH5Qys6Y6SxTJk7Z9ZyYFNIyy8jI/BSAdOsRZ6d4D0p6ui8qb90zY37BF/p9FRiOAR1WHOBsiUjoQVgpo+P1GdQBhXqzXSv/bMNhw7NPZ9Z5jcsnEeI+n5GaE5awJESURUQERbSGi7UR0Y+DxUiJ6jIi2EdE6IhoVeDyViJYS0VYiWkJEQwOPv0pEeUS0FsBj4cgWKimgYZaVl7kG5+hOJKIXa/VVAA9WncMuFlW+Ot6ne3erzmFjlQCeCOP6rgBQyczjmPlCBO6fD6hjZjf8bQKfDjz2LIB8Zr4IwJsAnmm3fAqA6cx8fxjzdZoUUGNkwz9fnhBf0rx7S1VnsBONfa5PKxa4mPmE6iw29VBKbkZzGNe3DcC3iOhRIspg5rp2z73d7vPJ7m3TALwV+Pp1AO37Iv/DDNfBpYAaICsvcz9kuh9xGs2716s6g900+k4MWV3z7/1m2JnazDaEMNvUuTDzHgATAuv+ExE91P7ps3x9NqZoRiIF1Dh/hL/1lRAAAPYdln6uBjjUtGvigcZtMqgovB5Iyc0I64xBRDQIQDMzvwH/bTHtZ6a6sd3nk1PZrcJXfcZvAlAUzjzhIAXUIFl5mccB/Fl1DmEOzN5mwCsNFAyy/ugnsxu8x2UO0fBYnJKbsbDjxTrNDWAdEW0G8HsAf2r3XC8i2grgXviniASAuwHcFnj8R4HnTIWYgzlaFqF47s6l8QB2AUhVHEUopnn2bvE2/Xuc6hx25qTYpmuH3l0Z44gdrTqLhekAJqTkZmyJ1AaJqBTAJGa2XKcjOQI1UFZeZhuA/1KdQ6ineUtkoIvBNPYmLap8NY6Z5dJJ6F6LZPG0OimgBsvKy/w7gCWqcwi1dO8had8XAQ3e48PW1Hy0l5nDev0uShwH8ECkN8rMqVY8+gSkgEbKnQBaVYcQajAzgxvltGKEHGzaOamscYfpBpxYwAMpuRk1qkNYiRTQCMjKy9yLUy+YiyjCWs1+AD1V54gma48WzG70nlijOoeFLAfwiuoQViMFNHIeA7BDdQgReZq3pEp1hmj0acWCCzXdt091DgvwALgjJTdDRpR2khTQCMnKy/QCuAPB3SQsbET37lcdISr52NttUeWrztM63oiveyQlN0PmWQ2BFNAIysrLXAngRdU5RGSxdlz63ypS7z2WuvZowR4ZVHRWuwE8ojqEVUkBjbxsANWqQ4jIYL35GKANV50jmpU17rj4YFOxDCr6Oob/1G2b6iBWJQU0wrLyMmthwo4awhiad59cgzOBNTX/mdXkrV2rOofJLEjJzShUHcLKpIAqkJWX+S6AfXHhDAAAEWVJREFU91TnEMbTPXvDOZuFCB0trFhwgcY+uSDtdwjAr1SHsDopoOrcAaBCdQhhLF2r7K06g/Dzsaf74op8YuZ61VkU0wH8KCU3Qzo2dZEUUEUCzeZvhozKtS1mzQtukwbyJlLnPTp8/dFPdnF0NwF/TE7dhocUUIWy8jKXAnhSdQ5hDN1XsQdAguoc4lQHGrdNLm/evVx1DkU2Aniow6VEUKSAqvcggC9UhxDhp3v2WLK/ZzRYdeTDWU2++nWqc0RYM4AfpORmyMTuYSIFVLGsvEwP/JPFtqjOIsJL95XFq84gzoo+LX85TWPfAdVBIui+lNyMPapD2IkUUBPIysssBvBr1TlEeLFeL/d/mpiXPT0+q3xdZ+YG1Vki4IOU3Axp4hJmUkBNIisv83kAH6nOIcJD104cAri/6hzi3Go9R0ZuPLZoh80HFVUBuF11CDuSAmoutwIoVZxBhIHuLTmkOoMIzr6GzVMrmkvsOqjIB+B7KbkZcj3eAFJATSQrL/MYgOsg10MtT/Ps86nOIIK38sj7Gc2+hvWqcxjg/pTcDLu+OVBOCqjJZOVlboacbrE81mrk9K21OBZWvDxGY61UdZAwyk/JzXhWdQg7kwJqQll5mW8CeFp1DhEa5rYGwDdadQ7ROV69LXlJ5es+Zm5UnSUMNgK4U3UIu5MCal6/AfC56hCi83Rv6R7I/y1LOuE5PGrTscXbVefoohoA16fkZrSqDmJ38p/cpLLyMn0AbgRwUHUW0Tmap8QORzBRa2/DF1Mrm/datdWdD8B3U3IzZL8RAVJATSwrL7MGwPUA5J2khei+8m6qM4iuKTr8z4wWX+MG1TlC8EBKbsYy1SGihRRQk8vKy9wIGVRkGcysg5ulgbz1ORZWvDxKZ61MdZBOeD0lN+Mp1SGiiRRQC8jKy3wDwP9TnUN0jLXqvQC6q84hus6jt/b8rPINDzM3qc4ShCUAfqo6RLSRAmoRWXmZfwKQpzqHODfNU1KtOoMInxOe6tFfHF+yVXWODmyBf9CQR3WQaCMF1FqyAHygOoQ4O917IEZ1BhFeJfUbp1U17zfroKJSAFem5GZE+yThSkgBtZCsvEwdwPcBrFSdRZwZ6yeGqM4gwq/o8HszW3xNG1XnOM0xAFek5GZUqQ4SraSAWkxWXmYrgHkAilVnEadiveEwoEsBtSEGOxdWvDRCZ80sPY5bAMxLyc3YrTpINJMCakFZeZknAFwBoEJ1FvEVzbMvmuaWjDoevbXXkqo3m5m5WXEUDcCNKbkZqxXniHpSQC0qKy/zIIArAdSpziL8dO/eNtUZhLGOt1Wdv+X455sVx7grJTfjP4ozCEgBtbSsvMxtAC4HIAMITED3VfdVnUEYb3f9+unVLaWqBhXdJxNjm4cUUIvLystcC+AySBFVitnXCnikgXyUWF797sxWrWlThDd7X0puhkwyYSJSQG1Aiqh6uu/gHgBxqnOIyGCwc2H5K8N01sojtEkpniYkBdQmpIiqpXtKjqvOICKrTW/us7Tq7UZmbjF4U1I8TUoKqI1IEVVH8x10qc4gIu9YW0Xa1hOFRp7KleJpYlJAbUaKqCJ6w0jVEYQau+rWzjjSUmbEoCIpniYnBdSG2hVRucUlAnTt6AEAfVTnEOosq35nRpvWHM7bW6R4WoAUUJsKFNEMAJWqs9id7imRhhZRjsExn5S/nKKz3tX/bz4At0jxtAYpoDYWuE90GqTtn6E0735WnUGo16Y3911W/XYtM7eGuIpGAFen5Ga8Fs5cwjhSQG0u0LFoBqQBvWFYOzpIdQZhDjWt5Rdsry3aEMJLjwC4JCU349NwZxLGkQIaBQK9cy8F8L7qLHbDekstoI1QnUOYx87a1TNrWg8t78RL9gKYnpKbEUrhFQpJAY0SgVlcbgDwguosdqJ795cAINU5hLl8XvX3aW1ay5YgFl0Pf/HcZ3QmEX7ELJdvos1zdy59EMCfVOewA0/jB8t07/45qnMI80lwJtXMG3KXz0GOgWdZZCGAG1JyM5oimUuEjxyBRqGsvMw/A7gZQKiDHUSA7qvspTqDMKdWralfYfU7x5n5TLP0/A3++TyleFqYHIFGsefuXDoZ/uuiMggmBMy6r632aQ8A6UIkzmpszxlFF/aamRH41gvgnpTcjDyVmUR4yBFoFMvKy1wHYBKANaqzWBH7KvdAiqfowI7alRlHWyuK4B9p+00pnvYhBTTKZeVlVgGYA+AlxVEsR/PuOao6g7CGz6vejvForRNTcjOKVGcR4SMFVCArL7MtKy/zdgC3AzjT9RpxBrq3NFZ1BmEJL+nQLhnx+LciNfWZiBC5BipO8dydSycBeA/AMNVZzK71xFNVAJ9thKUQrQB+8at3PnpZdRBhDDkCFafIysvcAGAigALVWcxM1+oqpHiKcygBMFOKp73JEag4q+fuXHoXgCcAJKrOYja+1g2rfC3Lp6vOIUzpJQC//NU7H8ktKjYnBVSc03N3Lk0H8CaAb6jOYiZtDe8sZ1/FLNU5hKkcA3D7r975SFpmRgk5hSvOKSsvsxjAVACPA5B3WwHsO9JPdQZhKp8BuEiKZ3SRI1ARtOfuXJoJIB9AiuosKjF7mtpq/5oAwKk6i1CuDcDvADz1q3c+kp1plJECKjrluTuX9gLwIvyN6aOS5in5wtv0HzmlLXYC+MGv3vkomKbxwobkFK4FENEcIvoo8PV8IsoOfP3q/2/v3mOkKu8wjn8fb8UIhdQ2RilNq7URBS9YxXt1o4laejGtXWv/8dLq6CZtmtGkxmrbVK2GUI11dCMk0oRoR8W0OCC1OkFRKyUGECwVvBa1Wi8syG2vv/5xDgER7XKYmTM7+3ySk9l95+w5vyFhn33POe/7SmpokHV0tq3t6Gw7H2gH3m7kuZtFf8+q9XnXYLnqIVmM4ViH5/DmAB1iImJORNycdx0dnW33A+NJJsUeVpcxBvre2C/vGiw3TwHHFMuV64rlihdjGOYcoHUkaT9JcyUtk7RCUnvafr2kxWnb3ZKUth8n6XlJSyVNlbRiJ8e8SNIdO2n/bdojbdh9uY7Otq6OzrYCcArwQqPOm6eICGLjoXnXYQ23FrgMOK1Yrvwz72KsOThA6+ts4K2IOCoiJpCs/wdwR0Qcl7btC0xJ2+8BLo+Io4H+wZ5E0lTgC8DFETHon6uVjs62Z0iGuVxLiy+RFv3/fRkYnXcd1lD3AeOL5cp0Pyhk23OA1tdy4CxJt0g6NSLWpe1nSFokaTnQBhwhaQwwKiL+nu5z7yDPcR0wOiIKkeMTYR2dbb0dnW03AROBx/Oqo976e1f/J+8arGFeBc4ulisXFsuVd/IuxpqPA7SOImIVMIkkSG9IL92OAO4Evh8RE4HpwIjdOM1i4FhJn9vtgmugo7PtpY7OtjOBC0h+AbWUgd5X/H+m9W0Efg1MKJYrf825Fmti/mVQR5IOAjZFxCySiQgmsS0s35M0knQ4SER0AR9Kmpy+f8EgTzMfuBmYK2lUzYrfTR2dbWXgMOAqkvtHLSH6PxjWY2BbXD/JH7RfLZYrvymWK5vyLsia2155F9DiJgJTJQ2QrER/RUR0SZoOrCAZBrJ4u/0vBaan+z8BrNvxgDsTEQ+k4TlH0rkRsbmmnyKjjs62HmBaqVC9B/gl0AHsk29V2cXAxndhwKvUtKZ5wNV+QMh2hSdSaCKSRkbEhvTrXwAHRsTPci6rZkqF6sHA74Af5F1LFn3dzy/q2/TY5P+/pw0hS4CriuVKNe9CbOhxgDaRdJjLNSRXBl4HLoqId/OtqvZKhepkklVeTsm7ll3R8+HsJwb6Xv9G3nVYTawheWp8lp+stawcoJabUqF6Jsml3SERSlu6SsuJ7ol512G75RXgFmBmsVzpybsYG9ocoJa7UqF6Mklv4Jy8a/kkEX3d3V23A3wm71osk5XATcB9xXKl4WOlrTU5QK1plArVSSRBeh6gnMv5iP7e11b0bnhoQt512C5bAtwIPORLtVZrDlBrOqVC9XCSJaIuoEmWDOvd+Lcn+nuWD4lLzQbAM8CNxXJlXt6FWOtygFrTKhWqXwGuAC4B9s+zlu51MxbFwHo/gdvceoHZQKlYrjyVdzHW+hyg1vRKheoIkqEvVwK5hNiWtb9/l2S+YWs+a0hWBZrhKfeskRygNqSk90mvBC4kmYi/7gb63/93z/o/fqkR57JBC+AxkmkxH/aDQZYHB6gNSaVCdQxwMVAAvlbPc/VtXvR035anT67nOWzQ1gIzgbuK5crqnGuxYc4BakNeqVA9Hvgh0A4cWOvjd6+/98nof/u0Wh/XBm0T8DDJCkXzPX7TmoUD1FpGqVDdAziDJEy/B4ypxXG3rL19NfR5Ee3G6gMeJQnNvxTLlQ0512P2MQ5Qa0mlQnUfkokZLgS+Rcb7pTGwZV33ujtH4ZWLGiGAp0hC88FiufJezvWYfSoHqLW8UqE6EjgLOJckVMcO9mf7e1Y+17vxkWPrVZuxgeRhoHnAvGK58mbO9ZgNmgPUhp1SoXokSZieC5zIpyzr17NhzoKB3pdOb1Bpw8VK4BGS0Fzoe5o2VDlAbVhLn+bd2js9ix16p1u67lpCbD4mj9payIfAk2zrZb6WbzlmteEANdtOOvvRqcCpEQMndXfdNg4YlXNZQ807JPcyF6bbMo/TtFbkADX7FNPap+xPMvvRCel2PDA616KaSy+wFHh261YsV17JtySzxnCAmu2Cae1TBHwZOCLdJqSv44ER+VVWdwPAy8ALwIrtXlf5HqYNVw5QsxqY1j5lT+BgPhqo49JtLLB3ftUNWpBcfl2TbqvZFpT/KpYrm3OszazpOEDN6mxa+5Q9gAOAL7ItVMel3x8AfDbdRqevtVy0u4/kIZ51wPp0ex94g21BuSb9/k33Js0GzwFq1mSmtU/Zh21hunUbTA92gCQstwblumK5sqledZoNdw5QMzOzDDw9mZmZWQYOUDMzswwcoGZmZhk4QM3MzDJwgJqZmWXgADUzM8vAAWpmuZA0Q9LhDTjPdxtxHht+PA7UzFqapJlAJSIezLsWay3ugZpZXUnaT9JcScskrZDUnrYvkPR1SZdIum27/X8i6db06z9Lek7SC5IuS9v2lDQzPdZyST9P2w+RND/df6GkwySdBHwbmCppqaRDGv8vYK1qr7wLMLOWdzbwVkR8E0DSjsvB3Q9cK+nqiOgFLgYuT9+7JCI+kLQvsFjSbJLVcMZGxIT0eGPSfe8GChGxWtJk4M6IaJM0B/dArQ4coGZWb8uBaZJuIQmyhdu/GREbJFWBKZJWAntHxPL07Z9KOi/9ehxwKPAicLCkPwBzgUcljQROAh6QtPXQtZyU3+xjfAnXzOoqIlYBk0iC9AZJ1+9ktxnARSS9z3sAJJ0OnAmcGBFHAUuAERGxFjgKWAAU0p/dA+iKiKO328bX83OZuQdqZnUl6SDgg4iYJakL+PGO+0TEIknjSIL2yLR5NLA2IjZJOgw4IT3e54GeiJgt6UVgVkSsl/SqpPMj4gEl3dAjI2IZyQo1o+r/SW24cQ/UzOptIvAPSUuBXwE3fMJ+9wNPpz1MgPnAXull3ZuBZ9P2scCC9HizgGvS9h8Bl0paRrIQ+HfS9j8BV0ta4oeIrJY8jMXMmoKkCnBrRDyedy1mg+EeqJnlStIYSauAzQ5PG0rcAzUzM8vAPVAzM7MMHKBmZmYZOEDNzMwycICamZll4AA1MzPLwAFqZmaWgQPUzMwsAweomZlZBg5QMzOzDBygZmZmGThAzczMMnCAmpmZZeAANTMzy8ABamZmloED1MzMLAMHqJmZWQYOUDMzswwcoGZmZhk4QM3MzDJwgJqZmWXgADUzM8vAAWpmZpaBA9TMzCwDB6iZmVkGDlAzM7MMHKBmZmYZOEDNzMwycICamZll4AA1MzPLwAFqZmaWgQPUzMwsAweomZlZBv8DbB5tIrZAPz8AAAAASUVORK5CYII=\n","text/plain":["<Figure size 576x576 with 1 Axes>"]},"metadata":{"tags":[]}}]},{"cell_type":"code","metadata":{"id":"r80yPEvU79rx","colab":{"base_uri":"https://localhost:8080/","height":115,"referenced_widgets":["21fbbe9bd6d94f308668ac1e4c2a51d0","f391632fced34067a6bc7f76034d8d38","d9bad87a40624bbe9a7b8e5021b6b878","6f869fe141f847a881532c74c5d8b269","35a5b5e6ff374b6a80e8677d695a5409","bed0bf19829f4957985091c7f308190c","43a1a07853414f1fbd41c1d7a7e2a7fd","66f31b78d9974a51bd28e01d073b72a7","8ed902a5a04e4bdaac8f961afe52b622","95a51d9d15734391a3895ac76b7a3676","395f10a793684087b6e3747154e17f8d","c905e0961cf84af2bec6a29ddc0d4237","b3652deb88d94a3e91bc0fa58b9dce54","94930151f87e495b974a7f40de2b1db4","4e66a8a8057e4232a3881aef3ac3afa9","a098193923044bfc89afdfa4faf8cc25"]},"executionInfo":{"status":"ok","timestamp":1625398193149,"user_tz":-180,"elapsed":2705,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"542fee56-a4bc-44c4-814e-8dd3c29eeb7f"},"source":["from transformers import BertTokenizerFast\n","tokenizer = BertTokenizerFast.from_pretrained(\"dbmdz/bert-base-turkish-uncased\", max_length=512)"],"execution_count":12,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"21fbbe9bd6d94f308668ac1e4c2a51d0","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=262620.0, style=ProgressStyle(descripti…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"8ed902a5a04e4bdaac8f961afe52b622","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=59.0, style=ProgressStyle(description_w…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"L7mT5FEU2Jyr","colab":{"base_uri":"https://localhost:8080/","height":1000,"referenced_widgets":["eda88132d04243bcb499af9e82a789ba","b9f606782e924fdd9a22ee6039ef76f2","ce256db990954d9f9782eae9a248b40b","4557e0d687a44a8ab109109094f3ebd6","5db915425bac4d9caba753797e9faf29","6613484a4d0c477fb4ec5a3d4da81a73","d391715c96a84d078976a10fab20265a","0479af3dea114a4b86c98cd104325344","f8532b3abd7e4243a8fb0ade8e0540e7","1886bd7f86534fcea494bef48439e7db","85c4358f5a6e411fbff53e3dfe3eb88d","cfe8a0c3575a49b6b298262ed7acf76d","4622af58d24f4eea8f67f49364118ccd","b42f5e933f494ea69f3dbaac391b5e23","f92aca32755f45c3ba2b8cab79d44eed","dd7628d863794892b9b1fdd7c5e5df42"]},"executionInfo":{"status":"ok","timestamp":1625398216241,"user_tz":-180,"elapsed":17953,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"b9133277-8b83-47f4-83ee-bdaa1ac55a3f"},"source":["from transformers import BertForSequenceClassification\n","model = BertForSequenceClassification.from_pretrained(\"dbmdz/bert-base-turkish-uncased\", num_labels=NUM_LABELS, id2label=id2label, label2id=label2id)\n","model.to(device)"],"execution_count":13,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"eda88132d04243bcb499af9e82a789ba","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=385.0, style=ProgressStyle(description_…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"f8532b3abd7e4243a8fb0ade8e0540e7","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=445018749.0, style=ProgressStyle(descri…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"stream","text":["Some weights of the model checkpoint at dbmdz/bert-base-turkish-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.decoder.bias', 'cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight']\n","- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n","- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n","Some weights of BertForSequenceClassification were not initialized from the model checkpoint at dbmdz/bert-base-turkish-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n","You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"],"name":"stderr"},{"output_type":"execute_result","data":{"text/plain":["BertForSequenceClassification(\n","  (bert): BertModel(\n","    (embeddings): BertEmbeddings(\n","      (word_embeddings): Embedding(32000, 768, padding_idx=0)\n","      (position_embeddings): Embedding(512, 768)\n","      (token_type_embeddings): Embedding(2, 768)\n","      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","      (dropout): Dropout(p=0.1, inplace=False)\n","    )\n","    (encoder): BertEncoder(\n","      (layer): ModuleList(\n","        (0): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (1): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (2): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (3): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (4): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (5): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (6): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (7): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (8): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (9): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (10): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (11): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","      )\n","    )\n","    (pooler): BertPooler(\n","      (dense): Linear(in_features=768, out_features=768, bias=True)\n","      (activation): Tanh()\n","    )\n","  )\n","  (dropout): Dropout(p=0.1, inplace=False)\n","  (classifier): Linear(in_features=768, out_features=7, bias=True)\n",")"]},"metadata":{"tags":[]},"execution_count":13}]},{"cell_type":"code","metadata":{"id":"X3ELK9wXh5z9","executionInfo":{"status":"ok","timestamp":1625398218952,"user_tz":-180,"elapsed":452,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}}},"source":[""],"execution_count":13,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"ZROZOxM9xi3l"},"source":["## Splitting data\n"]},{"cell_type":"code","metadata":{"id":"HM58CfgUZ0oK","executionInfo":{"status":"ok","timestamp":1625398219485,"user_tz":-180,"elapsed":3,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}}},"source":["SIZE= data.shape[0]\n","\n","train_texts= list(data.text[:SIZE//2])\n","val_texts=   list(data.text[SIZE//2:(3*SIZE)//4 ])\n","test_texts=  list(data.text[(3*SIZE)//4:])\n","\n","train_labels= list(data.labels[:SIZE//2])\n","val_labels=   list(data.labels[SIZE//2:(3*SIZE)//4])\n","test_labels=  list(data.labels[(3*SIZE)//4:])"],"execution_count":14,"outputs":[]},{"cell_type":"code","metadata":{"id":"vwe5Bt8CfhdU","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625398221030,"user_tz":-180,"elapsed":7,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"619c0354-0462-485e-804c-b5df86b7d713"},"source":["len(train_texts), len(val_texts), len(test_texts)"],"execution_count":15,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(2450, 1225, 1225)"]},"metadata":{"tags":[]},"execution_count":15}]},{"cell_type":"code","metadata":{"id":"snOgiQe2mbpx","executionInfo":{"status":"ok","timestamp":1625398225432,"user_tz":-180,"elapsed":4406,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}}},"source":["train_encodings = tokenizer(train_texts, truncation=True, padding=True)\n","val_encodings  = tokenizer(val_texts, truncation=True, padding=True)\n","test_encodings = tokenizer(test_texts, truncation=True, padding=True)"],"execution_count":16,"outputs":[]},{"cell_type":"code","metadata":{"id":"TvN9Siw65H9G","executionInfo":{"status":"ok","timestamp":1625398226853,"user_tz":-180,"elapsed":1,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}}},"source":["from torch.utils.data import Dataset\n","class MyDataset(Dataset):\n","    def __init__(self, encodings, labels):\n","        self.encodings = encodings\n","        self.labels = labels\n","    def __getitem__(self, idx):\n","        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n","        item['labels'] = torch.tensor(self.labels[idx])\n","        return item\n","    def __len__(self):\n","        return len(self.labels)"],"execution_count":17,"outputs":[]},{"cell_type":"code","metadata":{"id":"Vx3u-9ljtmM_","executionInfo":{"status":"ok","timestamp":1625398229008,"user_tz":-180,"elapsed":412,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}}},"source":["train_dataset = MyDataset(train_encodings, train_labels)\n","val_dataset = MyDataset(val_encodings, val_labels)\n","test_dataset = MyDataset(test_encodings, test_labels)"],"execution_count":18,"outputs":[]},{"cell_type":"code","metadata":{"id":"earrg8TiBojU","executionInfo":{"status":"ok","timestamp":1625398230906,"user_tz":-180,"elapsed":4,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}}},"source":[""],"execution_count":18,"outputs":[]},{"cell_type":"code","metadata":{"id":"SAu_MvIdp_Gq","executionInfo":{"status":"ok","timestamp":1625398231275,"user_tz":-180,"elapsed":2,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}}},"source":[""],"execution_count":18,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"-dXcl4LWBsJy"},"source":["## Training with Trainer Class"]},{"cell_type":"code","metadata":{"id":"jHP9LR_QsytZ","executionInfo":{"status":"ok","timestamp":1625398235958,"user_tz":-180,"elapsed":2440,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}}},"source":["from transformers import TrainingArguments, Trainer"],"execution_count":19,"outputs":[]},{"cell_type":"code","metadata":{"id":"PjcZ_K87l9CX","executionInfo":{"status":"ok","timestamp":1625398235959,"user_tz":-180,"elapsed":6,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}}},"source":["from sklearn.metrics import accuracy_score, precision_recall_fscore_support \n","def compute_metrics(pred): \n","    labels = pred.label_ids \n","    preds = pred.predictions.argmax(-1) \n","    precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average='macro') \n","    acc = accuracy_score(labels, preds) \n","    return { \n","        'Accuracy': acc, \n","        'F1': f1, \n","        'Precision': precision, \n","        'Recall': recall \n","    } "],"execution_count":20,"outputs":[]},{"cell_type":"code","metadata":{"id":"zMiPOIVAmYI2","executionInfo":{"status":"ok","timestamp":1625398239957,"user_tz":-180,"elapsed":1491,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}}},"source":["training_args = TrainingArguments(\n","    # The output directory where the model predictions and checkpoints will be written\n","    output_dir='./TTC4900Model', \n","    do_train=True,\n","    do_eval=True,\n","    #  The number of epochs, defaults to 3.0 \n","    num_train_epochs=3,              \n","    per_device_train_batch_size=16,  \n","    per_device_eval_batch_size=32,\n","    # Number of steps used for a linear warmup\n","    warmup_steps=100,                \n","    weight_decay=0.01,\n","    logging_strategy='steps',\n","   # TensorBoard log directory                 \n","    logging_dir='./multi-class-logs',            \n","    logging_steps=50,\n","    evaluation_strategy=\"steps\",\n","    eval_steps=50,\n","    save_strategy=\"epoch\", \n","    fp16=True,\n","    load_best_model_at_end=True\n",")"],"execution_count":21,"outputs":[]},{"cell_type":"code","metadata":{"id":"8Ajz99mwj-OL","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625398241364,"user_tz":-180,"elapsed":5,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"d3ee309d-d48a-4ed1-d9f6-ed48fcb3102f"},"source":["trainer = Trainer(\n","    # the pre-trained model that will be fine-tuned \n","    model=model,\n","     # training arguments that we defined above                        \n","    args=training_args,                 \n","    train_dataset=train_dataset,         \n","    eval_dataset=val_dataset,            \n","    compute_metrics= compute_metrics\n",")"],"execution_count":22,"outputs":[{"output_type":"stream","text":["Using amp fp16 backend\n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000},"id":"1WRhCH-Hj-RT","executionInfo":{"status":"ok","timestamp":1625398945390,"user_tz":-180,"elapsed":701522,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"fa531859-8922-4ced-9796-2301c3ebf091"},"source":["trainer.train()"],"execution_count":23,"outputs":[{"output_type":"stream","text":["***** Running training *****\n","  Num examples = 2450\n","  Num Epochs = 3\n","  Instantaneous batch size per device = 16\n","  Total train batch size (w. parallel, distributed & accumulation) = 16\n","  Gradient Accumulation steps = 1\n","  Total optimization steps = 462\n"],"name":"stderr"},{"output_type":"display_data","data":{"text/html":["\n","    <div>\n","      \n","      <progress value='462' max='462' style='width:300px; height:20px; vertical-align: middle;'></progress>\n","      [462/462 11:39, Epoch 3/3]\n","    </div>\n","    <table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: left;\">\n","      <th>Step</th>\n","      <th>Training Loss</th>\n","      <th>Validation Loss</th>\n","      <th>Accuracy</th>\n","      <th>F1</th>\n","      <th>Precision</th>\n","      <th>Recall</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <td>50</td>\n","      <td>1.850700</td>\n","      <td>1.507407</td>\n","      <td>0.624490</td>\n","      <td>0.608790</td>\n","      <td>0.740980</td>\n","      <td>0.627075</td>\n","    </tr>\n","    <tr>\n","      <td>100</td>\n","      <td>0.733500</td>\n","      <td>0.365703</td>\n","      <td>0.895510</td>\n","      <td>0.896049</td>\n","      <td>0.906131</td>\n","      <td>0.896706</td>\n","    </tr>\n","    <tr>\n","      <td>150</td>\n","      <td>0.361300</td>\n","      <td>0.306502</td>\n","      <td>0.908571</td>\n","      <td>0.908585</td>\n","      <td>0.910181</td>\n","      <td>0.908059</td>\n","    </tr>\n","    <tr>\n","      <td>200</td>\n","      <td>0.262900</td>\n","      <td>0.318767</td>\n","      <td>0.905306</td>\n","      <td>0.905354</td>\n","      <td>0.909020</td>\n","      <td>0.905336</td>\n","    </tr>\n","    <tr>\n","      <td>250</td>\n","      <td>0.159000</td>\n","      <td>0.324903</td>\n","      <td>0.915102</td>\n","      <td>0.915583</td>\n","      <td>0.918657</td>\n","      <td>0.915158</td>\n","    </tr>\n","    <tr>\n","      <td>300</td>\n","      <td>0.227400</td>\n","      <td>0.331741</td>\n","      <td>0.920000</td>\n","      <td>0.918919</td>\n","      <td>0.921991</td>\n","      <td>0.919482</td>\n","    </tr>\n","    <tr>\n","      <td>350</td>\n","      <td>0.078200</td>\n","      <td>0.351717</td>\n","      <td>0.921633</td>\n","      <td>0.921153</td>\n","      <td>0.921619</td>\n","      <td>0.921414</td>\n","    </tr>\n","    <tr>\n","      <td>400</td>\n","      <td>0.087700</td>\n","      <td>0.338882</td>\n","      <td>0.919184</td>\n","      <td>0.919011</td>\n","      <td>0.919340</td>\n","      <td>0.918896</td>\n","    </tr>\n","    <tr>\n","      <td>450</td>\n","      <td>0.084000</td>\n","      <td>0.326685</td>\n","      <td>0.923265</td>\n","      <td>0.922903</td>\n","      <td>0.922840</td>\n","      <td>0.923093</td>\n","    </tr>\n","  </tbody>\n","</table><p>"],"text/plain":["<IPython.core.display.HTML object>"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["/usr/local/lib/python3.7/dist-packages/transformers/trainer.py:1299: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n","  args.max_grad_norm,\n","***** Running Evaluation *****\n","  Num examples = 1225\n","  Batch size = 32\n","Saving model checkpoint to ./TTC4900Model/checkpoint-50\n","Configuration saved in ./TTC4900Model/checkpoint-50/config.json\n","Model weights saved in ./TTC4900Model/checkpoint-50/pytorch_model.bin\n","***** Running Evaluation *****\n","  Num examples = 1225\n","  Batch size = 32\n","Saving model checkpoint to ./TTC4900Model/checkpoint-100\n","Configuration saved in ./TTC4900Model/checkpoint-100/config.json\n","Model weights saved in ./TTC4900Model/checkpoint-100/pytorch_model.bin\n","***** Running Evaluation *****\n","  Num examples = 1225\n","  Batch size = 32\n","Saving model checkpoint to ./TTC4900Model/checkpoint-150\n","Configuration saved in ./TTC4900Model/checkpoint-150/config.json\n","Model weights saved in ./TTC4900Model/checkpoint-150/pytorch_model.bin\n","Saving model checkpoint to ./TTC4900Model/checkpoint-154\n","Configuration saved in ./TTC4900Model/checkpoint-154/config.json\n","Model weights saved in ./TTC4900Model/checkpoint-154/pytorch_model.bin\n","***** Running Evaluation *****\n","  Num examples = 1225\n","  Batch size = 32\n","Saving model checkpoint to ./TTC4900Model/checkpoint-200\n","Configuration saved in ./TTC4900Model/checkpoint-200/config.json\n","Model weights saved in ./TTC4900Model/checkpoint-200/pytorch_model.bin\n","***** Running Evaluation *****\n","  Num examples = 1225\n","  Batch size = 32\n","Saving model checkpoint to ./TTC4900Model/checkpoint-250\n","Configuration saved in ./TTC4900Model/checkpoint-250/config.json\n","Model weights saved in ./TTC4900Model/checkpoint-250/pytorch_model.bin\n","***** Running Evaluation *****\n","  Num examples = 1225\n","  Batch size = 32\n","Saving model checkpoint to ./TTC4900Model/checkpoint-300\n","Configuration saved in ./TTC4900Model/checkpoint-300/config.json\n","Model weights saved in ./TTC4900Model/checkpoint-300/pytorch_model.bin\n","Saving model checkpoint to ./TTC4900Model/checkpoint-308\n","Configuration saved in ./TTC4900Model/checkpoint-308/config.json\n","Model weights saved in ./TTC4900Model/checkpoint-308/pytorch_model.bin\n","***** Running Evaluation *****\n","  Num examples = 1225\n","  Batch size = 32\n","Saving model checkpoint to ./TTC4900Model/checkpoint-350\n","Configuration saved in ./TTC4900Model/checkpoint-350/config.json\n","Model weights saved in ./TTC4900Model/checkpoint-350/pytorch_model.bin\n","***** Running Evaluation *****\n","  Num examples = 1225\n","  Batch size = 32\n","Saving model checkpoint to ./TTC4900Model/checkpoint-400\n","Configuration saved in ./TTC4900Model/checkpoint-400/config.json\n","Model weights saved in ./TTC4900Model/checkpoint-400/pytorch_model.bin\n","***** Running Evaluation *****\n","  Num examples = 1225\n","  Batch size = 32\n","Saving model checkpoint to ./TTC4900Model/checkpoint-450\n","Configuration saved in ./TTC4900Model/checkpoint-450/config.json\n","Model weights saved in ./TTC4900Model/checkpoint-450/pytorch_model.bin\n","Saving model checkpoint to ./TTC4900Model/checkpoint-462\n","Configuration saved in ./TTC4900Model/checkpoint-462/config.json\n","Model weights saved in ./TTC4900Model/checkpoint-462/pytorch_model.bin\n","\n","\n","Training completed. Do not forget to share your model on huggingface.co/models =)\n","\n","\n","Loading best model from ./TTC4900Model/checkpoint-150 (score: 0.3065018653869629).\n"],"name":"stderr"},{"output_type":"execute_result","data":{"text/plain":["TrainOutput(global_step=462, training_loss=0.4197033045095799, metrics={'train_runtime': 700.7851, 'train_samples_per_second': 10.488, 'train_steps_per_second': 0.659, 'total_flos': 2497772677478400.0, 'train_loss': 0.4197033045095799, 'epoch': 3.0})"]},"metadata":{"tags":[]},"execution_count":23}]},{"cell_type":"code","metadata":{"id":"FN2i7kRIj-UQ","colab":{"base_uri":"https://localhost:8080/","height":315},"executionInfo":{"status":"ok","timestamp":1625399043817,"user_tz":-180,"elapsed":85330,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"71974f34-e461-4999-9ab8-558783167509"},"source":["q=[trainer.evaluate(eval_dataset=data) for data in [train_dataset, val_dataset, test_dataset]]\n","pd.DataFrame(q, index=[\"train\",\"val\",\"test\"]).iloc[:,:5]"],"execution_count":24,"outputs":[{"output_type":"stream","text":["***** Running Evaluation *****\n","  Num examples = 2450\n","  Batch size = 32\n"],"name":"stderr"},{"output_type":"display_data","data":{"text/html":["\n","    <div>\n","      \n","      <progress value='155' max='77' style='width:300px; height:20px; vertical-align: middle;'></progress>\n","      [77/77 01:24]\n","    </div>\n","    "],"text/plain":["<IPython.core.display.HTML object>"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["***** Running Evaluation *****\n","  Num examples = 1225\n","  Batch size = 32\n","***** Running Evaluation *****\n","  Num examples = 1225\n","  Batch size = 32\n"],"name":"stderr"},{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>eval_loss</th>\n","      <th>eval_Accuracy</th>\n","      <th>eval_F1</th>\n","      <th>eval_Precision</th>\n","      <th>eval_Recall</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>train</th>\n","      <td>0.197921</td>\n","      <td>0.941633</td>\n","      <td>0.941545</td>\n","      <td>0.942952</td>\n","      <td>0.941139</td>\n","    </tr>\n","    <tr>\n","      <th>val</th>\n","      <td>0.306502</td>\n","      <td>0.908571</td>\n","      <td>0.908585</td>\n","      <td>0.910181</td>\n","      <td>0.908059</td>\n","    </tr>\n","    <tr>\n","      <th>test</th>\n","      <td>0.318877</td>\n","      <td>0.908571</td>\n","      <td>0.908620</td>\n","      <td>0.908974</td>\n","      <td>0.909747</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["       eval_loss  eval_Accuracy   eval_F1  eval_Precision  eval_Recall\n","train   0.197921       0.941633  0.941545        0.942952     0.941139\n","val     0.306502       0.908571  0.908585        0.910181     0.908059\n","test    0.318877       0.908571  0.908620        0.908974     0.909747"]},"metadata":{"tags":[]},"execution_count":24}]},{"cell_type":"code","metadata":{"id":"H-mMU2rlj-Xa","executionInfo":{"status":"ok","timestamp":1625399043819,"user_tz":-180,"elapsed":12,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}}},"source":[""],"execution_count":24,"outputs":[]},{"cell_type":"code","metadata":{"id":"I0guPT0jYJth","executionInfo":{"status":"ok","timestamp":1625399048192,"user_tz":-180,"elapsed":392,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}}},"source":["from transformers import DistilBertForSequenceClassification, DistilBertTokenizerFast"],"execution_count":25,"outputs":[]},{"cell_type":"code","metadata":{"id":"7g6_tkNXj-fs","executionInfo":{"status":"ok","timestamp":1625399048604,"user_tz":-180,"elapsed":3,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}}},"source":["def predict(text):\n","    inputs = tokenizer(text, padding=True, truncation=True, max_length=512, return_tensors=\"pt\").to(\"cuda\")\n","    outputs = model(**inputs)\n","    probs = outputs[0].softmax(1)\n","    return probs, probs.argmax(),model.config.id2label[probs.argmax().item()]"],"execution_count":26,"outputs":[]},{"cell_type":"code","metadata":{"id":"mac8gGgWmYNn","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625399050044,"user_tz":-180,"elapsed":4,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"f6038697-d659-44b8-b0fe-946c9ce8e7ac"},"source":["# Example #1\n","text = \"Fenerbahçeli futbolcular kısa paslarla hazırlık çalışması yaptılar\"\n","predict(text)"],"execution_count":27,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(tensor([[0.0043, 0.0068, 0.0075, 0.0047, 0.0077, 0.9663, 0.0026]],\n","        device='cuda:0', grad_fn=<SoftmaxBackward>),\n"," tensor(5, device='cuda:0'),\n"," 'spor')"]},"metadata":{"tags":[]},"execution_count":27}]},{"cell_type":"code","metadata":{"id":"z6cmFfkUDobU","executionInfo":{"status":"ok","timestamp":1625399051931,"user_tz":-180,"elapsed":8,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}}},"source":[""],"execution_count":27,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"WWoJjzP6aYlc"},"source":["## Save and Re-Load saved model for inference"]},{"cell_type":"code","metadata":{"id":"1nU9QTjil_pM","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625399057943,"user_tz":-180,"elapsed":4589,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"2810f8bc-3447-43e5-fe3f-e080a197b4bc"},"source":["!pip install transformers"],"execution_count":28,"outputs":[{"output_type":"stream","text":["Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.8.2)\n","Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n","Requirement already satisfied: huggingface-hub==0.0.12 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.12)\n","Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n","Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (4.5.0)\n","Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n","Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.10.3)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n","Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n","Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from transformers) (3.13)\n","Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n","Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.45)\n","Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from huggingface-hub==0.0.12->transformers) (3.7.4.3)\n","Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n","Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.5.30)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n","Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n","Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n","Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"aF7GVSKSYBPw","executionInfo":{"status":"ok","timestamp":1625399060094,"user_tz":-180,"elapsed":2155,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"a0bcea98-9f37-460f-bf56-fa7604edd970"},"source":["# saving the fine tuned model & tokenizer\n","model_path = \"turkish-text-classification-model\"\n","trainer.save_model(model_path)\n","tokenizer.save_pretrained(model_path)"],"execution_count":29,"outputs":[{"output_type":"stream","text":["Saving model checkpoint to turkish-text-classification-model\n","Configuration saved in turkish-text-classification-model/config.json\n","Model weights saved in turkish-text-classification-model/pytorch_model.bin\n","tokenizer config file saved in turkish-text-classification-model/tokenizer_config.json\n","Special tokens file saved in turkish-text-classification-model/special_tokens_map.json\n"],"name":"stderr"},{"output_type":"execute_result","data":{"text/plain":["('turkish-text-classification-model/tokenizer_config.json',\n"," 'turkish-text-classification-model/special_tokens_map.json',\n"," 'turkish-text-classification-model/vocab.txt',\n"," 'turkish-text-classification-model/added_tokens.json',\n"," 'turkish-text-classification-model/tokenizer.json')"]},"metadata":{"tags":[]},"execution_count":29}]},{"cell_type":"code","metadata":{"id":"EA_eNKP_an4t"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"qEYQdBL8Tdkm","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625399066951,"user_tz":-180,"elapsed":2937,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"0e6b93fa-32ab-481d-e676-94dd051acec6"},"source":["model_path = \"turkish-text-classification-model\"\n","from transformers import pipeline, BertForSequenceClassification, BertTokenizerFast\n","model = BertForSequenceClassification.from_pretrained(model_path)\n","tokenizer= BertTokenizerFast.from_pretrained(model_path)\n","nlp= pipeline(\"sentiment-analysis\", model=model, tokenizer=tokenizer)"],"execution_count":30,"outputs":[{"output_type":"stream","text":["loading configuration file turkish-text-classification-model/config.json\n","Model config BertConfig {\n","  \"_name_or_path\": \"dbmdz/bert-base-turkish-uncased\",\n","  \"architectures\": [\n","    \"BertForSequenceClassification\"\n","  ],\n","  \"attention_probs_dropout_prob\": 0.1,\n","  \"gradient_checkpointing\": false,\n","  \"hidden_act\": \"gelu\",\n","  \"hidden_dropout_prob\": 0.1,\n","  \"hidden_size\": 768,\n","  \"id2label\": {\n","    \"0\": \"teknoloji\",\n","    \"1\": \"ekonomi\",\n","    \"2\": \"saglik\",\n","    \"3\": \"siyaset\",\n","    \"4\": \"kultur\",\n","    \"5\": \"spor\",\n","    \"6\": \"dunya\"\n","  },\n","  \"initializer_range\": 0.02,\n","  \"intermediate_size\": 3072,\n","  \"label2id\": {\n","    \"dunya\": 6,\n","    \"ekonomi\": 1,\n","    \"kultur\": 4,\n","    \"saglik\": 2,\n","    \"siyaset\": 3,\n","    \"spor\": 5,\n","    \"teknoloji\": 0\n","  },\n","  \"layer_norm_eps\": 1e-12,\n","  \"max_position_embeddings\": 512,\n","  \"model_type\": \"bert\",\n","  \"num_attention_heads\": 12,\n","  \"num_hidden_layers\": 12,\n","  \"pad_token_id\": 0,\n","  \"position_embedding_type\": \"absolute\",\n","  \"problem_type\": \"single_label_classification\",\n","  \"transformers_version\": \"4.8.2\",\n","  \"type_vocab_size\": 2,\n","  \"use_cache\": true,\n","  \"vocab_size\": 32000\n","}\n","\n","loading weights file turkish-text-classification-model/pytorch_model.bin\n","All model checkpoint weights were used when initializing BertForSequenceClassification.\n","\n","All the weights of BertForSequenceClassification were initialized from the model checkpoint at turkish-text-classification-model.\n","If your task is similar to the task the model of the checkpoint was trained on, you can already use BertForSequenceClassification for predictions without further training.\n","Didn't find file turkish-text-classification-model/added_tokens.json. We won't load it.\n","loading file turkish-text-classification-model/vocab.txt\n","loading file turkish-text-classification-model/tokenizer.json\n","loading file None\n","loading file turkish-text-classification-model/special_tokens_map.json\n","loading file turkish-text-classification-model/tokenizer_config.json\n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"id":"Z0-NzK4AEhBg"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"zyiJkIvzcdgC","executionInfo":{"status":"ok","timestamp":1625399077112,"user_tz":-180,"elapsed":533,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"ddd71497-8990-4cde-95b9-99882f33bb17"},"source":["nlp(\"Sinemada hangi filmler oynuyor bugün\")"],"execution_count":31,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[{'label': 'kultur', 'score': 0.897723913192749}]"]},"metadata":{"tags":[]},"execution_count":31}]},{"cell_type":"code","metadata":{"id":"RPLqmWPMGc53","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625399079590,"user_tz":-180,"elapsed":515,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"4070ccb8-9f2d-491e-b96e-0e8202777c07"},"source":["nlp(\"Dolar ve Euro bugün yurtiçi piyasalarda yükseldi\")"],"execution_count":32,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[{'label': 'ekonomi', 'score': 0.9639127254486084}]"]},"metadata":{"tags":[]},"execution_count":32}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"5hc65L_mQcVf","executionInfo":{"status":"ok","timestamp":1625399082230,"user_tz":-180,"elapsed":632,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"43a2bda2-c915-47b5-f29c-8319bf76deeb"},"source":["nlp(\"Bayern Münih ile Barcelona bugün karşı karşıya geliyor. Maçı İngiliz hakem James Watts yönetecek!\")"],"execution_count":33,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[{'label': 'spor', 'score': 0.9791778922080994}]"]},"metadata":{"tags":[]},"execution_count":33}]},{"cell_type":"code","metadata":{"id":"hOqw_MoAVbgB"},"source":[""],"execution_count":null,"outputs":[]}]}